diff options
99 files changed, 2499 insertions, 781 deletions
diff --git a/.ci/scripts/android/build.sh b/.ci/scripts/android/build.sh index d135af029..98593017f 100755 --- a/.ci/scripts/android/build.sh +++ b/.ci/scripts/android/build.sh @@ -6,7 +6,12 @@ export NDK_CCACHE="$(which ccache)" ccache -s -BUILD_FLAVOR=mainline +BUILD_FLAVOR="mainline" + +BUILD_TYPE="release" +if [ "${GITHUB_REPOSITORY}" == "yuzu-emu/yuzu" ]; then + BUILD_TYPE="relWithDebInfo" +fi if [ ! -z "${ANDROID_KEYSTORE_B64}" ]; then export ANDROID_KEYSTORE_FILE="${GITHUB_WORKSPACE}/ks.jks" @@ -15,7 +20,7 @@ fi cd src/android chmod +x ./gradlew -./gradlew "assemble${BUILD_FLAVOR}Release" "bundle${BUILD_FLAVOR}Release" +./gradlew "assemble${BUILD_FLAVOR}${BUILD_TYPE}" "bundle${BUILD_FLAVOR}${BUILD_TYPE}" ccache -s diff --git a/.ci/scripts/android/upload.sh b/.ci/scripts/android/upload.sh index 5f8ca73c0..26b1a7efa 100755 --- a/.ci/scripts/android/upload.sh +++ b/.ci/scripts/android/upload.sh @@ -7,9 +7,16 @@ REV_NAME="yuzu-${GITDATE}-${GITREV}" -BUILD_FLAVOR=mainline +BUILD_FLAVOR="mainline" -cp src/android/app/build/outputs/apk/"${BUILD_FLAVOR}/release/app-${BUILD_FLAVOR}-release.apk" \ +BUILD_TYPE_LOWER="release" +BUILD_TYPE_UPPER="Release" +if [ "${GITHUB_REPOSITORY}" == "yuzu-emu/yuzu" ]; then + BUILD_TYPE_LOWER="relWithDebInfo" + BUILD_TYPE_UPPER="RelWithDebInfo" +fi + +cp src/android/app/build/outputs/apk/"${BUILD_FLAVOR}/${BUILD_TYPE_LOWER}/app-${BUILD_FLAVOR}-${BUILD_TYPE_LOWER}.apk" \ "artifacts/${REV_NAME}.apk" -cp src/android/app/build/outputs/bundle/"${BUILD_FLAVOR}Release"/"app-${BUILD_FLAVOR}-release.aab" \ +cp src/android/app/build/outputs/bundle/"${BUILD_FLAVOR}${BUILD_TYPE_UPPER}"/"app-${BUILD_FLAVOR}-${BUILD_TYPE_LOWER}.aab" \ "artifacts/${REV_NAME}.aab" diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/IntSetting.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/IntSetting.kt index 21e4e1afd..df760440f 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/IntSetting.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/IntSetting.kt @@ -18,7 +18,8 @@ enum class IntSetting(override val key: String) : AbstractIntSetting { RENDERER_ANTI_ALIASING("anti_aliasing"), RENDERER_SCREEN_LAYOUT("screen_layout"), RENDERER_ASPECT_RATIO("aspect_ratio"), - AUDIO_OUTPUT_ENGINE("output_engine"); + AUDIO_OUTPUT_ENGINE("output_engine"), + MAX_ANISOTROPY("max_anisotropy"); override fun getInt(needsGlobal: Boolean): Int = NativeConfig.getInt(key, needsGlobal) diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt index 2e97aee2c..12f7aa1ab 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt @@ -245,6 +245,15 @@ abstract class SettingsItem( ) put( SingleChoiceSetting( + IntSetting.MAX_ANISOTROPY, + R.string.anisotropic_filtering, + R.string.anisotropic_filtering_description, + R.array.anisoEntries, + R.array.anisoValues + ) + ) + put( + SingleChoiceSetting( IntSetting.AUDIO_OUTPUT_ENGINE, R.string.audio_output_engine, 0, @@ -298,6 +307,7 @@ abstract class SettingsItem( override val key: String = FASTMEM_COMBINED override val isRuntimeModifiable: Boolean = false + override val pairedSettingKey = BooleanSetting.CPU_DEBUG_MODE.key override val defaultValue: Boolean = true override val isSwitchable: Boolean = true override var global: Boolean diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt index a7e965589..db1a1076c 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt @@ -149,6 +149,7 @@ class SettingsFragmentPresenter( add(IntSetting.RENDERER_VSYNC.key) add(IntSetting.RENDERER_SCALING_FILTER.key) add(IntSetting.RENDERER_ANTI_ALIASING.key) + add(IntSetting.MAX_ANISOTROPY.key) add(IntSetting.RENDERER_SCREEN_LAYOUT.key) add(IntSetting.RENDERER_ASPECT_RATIO.key) add(BooleanSetting.PICTURE_IN_PICTURE.key) diff --git a/src/android/app/src/main/res/layout/list_item_setting_switch.xml b/src/android/app/src/main/res/layout/list_item_setting_switch.xml index 5cb84182e..1c08e2e1b 100644 --- a/src/android/app/src/main/res/layout/list_item_setting_switch.xml +++ b/src/android/app/src/main/res/layout/list_item_setting_switch.xml @@ -24,7 +24,7 @@ android:layout_width="0dp" android:layout_height="wrap_content" android:layout_marginEnd="24dp" - android:gravity="center_vertical" + android:layout_gravity="center_vertical" android:orientation="vertical" android:layout_weight="1"> diff --git a/src/android/app/src/main/res/values/arrays.xml b/src/android/app/src/main/res/values/arrays.xml index e3915ef4f..c882a8e62 100644 --- a/src/android/app/src/main/res/values/arrays.xml +++ b/src/android/app/src/main/res/values/arrays.xml @@ -267,4 +267,21 @@ <item>3</item> </integer-array> + <string-array name="anisoEntries"> + <item>@string/auto</item> + <item>@string/slider_default</item> + <item>@string/multiplier_two</item> + <item>@string/multiplier_four</item> + <item>@string/multiplier_eight</item> + <item>@string/multiplier_sixteen</item> + </string-array> + <integer-array name="anisoValues"> + <item>0</item> + <item>1</item> + <item>2</item> + <item>3</item> + <item>4</item> + <item>5</item> + </integer-array> + </resources> diff --git a/src/android/app/src/main/res/values/strings.xml b/src/android/app/src/main/res/values/strings.xml index 0b80b04a4..4d5c268fe 100644 --- a/src/android/app/src/main/res/values/strings.xml +++ b/src/android/app/src/main/res/values/strings.xml @@ -225,6 +225,8 @@ <string name="renderer_reactive_flushing_description">Improves rendering accuracy in some games at the cost of performance.</string> <string name="use_disk_shader_cache">Disk shader cache</string> <string name="use_disk_shader_cache_description">Reduces stuttering by locally storing and loading generated shaders.</string> + <string name="anisotropic_filtering">Anisotropic filtering</string> + <string name="anisotropic_filtering_description">Improves the quality of textures when viewed at oblique angles</string> <!-- Debug settings strings --> <string name="cpu">CPU</string> @@ -506,6 +508,12 @@ <string name="oboe">oboe</string> <string name="cubeb">cubeb</string> + <!-- Anisotropic filtering options --> + <string name="multiplier_two">2x</string> + <string name="multiplier_four">4x</string> + <string name="multiplier_eight">8x</string> + <string name="multiplier_sixteen">16x</string> + <!-- Black backgrounds theme --> <string name="use_black_backgrounds">Black backgrounds</string> <string name="use_black_backgrounds_description">When using the dark theme, apply black backgrounds.</string> diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index b58a7073f..8c57d47c6 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -64,6 +64,8 @@ add_library(common STATIC fs/path_util.cpp fs/path_util.h hash.h + heap_tracker.cpp + heap_tracker.h hex_util.cpp hex_util.h host_memory.cpp diff --git a/src/common/assert.cpp b/src/common/assert.cpp index 6026b7dc2..e2c2cade3 100644 --- a/src/common/assert.cpp +++ b/src/common/assert.cpp @@ -3,16 +3,19 @@ #include "common/assert.h" #include "common/common_funcs.h" +#include "common/logging/backend.h" #include "common/settings.h" void assert_fail_impl() { if (Settings::values.use_debug_asserts) { + Common::Log::Stop(); Crash(); } } [[noreturn]] void unreachable_impl() { + Common::Log::Stop(); Crash(); throw std::runtime_error("Unreachable code"); } diff --git a/src/common/heap_tracker.cpp b/src/common/heap_tracker.cpp new file mode 100644 index 000000000..683208795 --- /dev/null +++ b/src/common/heap_tracker.cpp @@ -0,0 +1,281 @@ +// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include <fstream> +#include <vector> + +#include "common/heap_tracker.h" +#include "common/logging/log.h" + +namespace Common { + +namespace { + +s64 GetMaxPermissibleResidentMapCount() { + // Default value. + s64 value = 65530; + + // Try to read how many mappings we can make. + std::ifstream s("/proc/sys/vm/max_map_count"); + s >> value; + + // Print, for debug. + LOG_INFO(HW_Memory, "Current maximum map count: {}", value); + + // Allow 20000 maps for other code and to account for split inaccuracy. + return std::max<s64>(value - 20000, 0); +} + +} // namespace + +HeapTracker::HeapTracker(Common::HostMemory& buffer) + : m_buffer(buffer), m_max_resident_map_count(GetMaxPermissibleResidentMapCount()) {} +HeapTracker::~HeapTracker() = default; + +void HeapTracker::Map(size_t virtual_offset, size_t host_offset, size_t length, + MemoryPermission perm, bool is_separate_heap) { + // When mapping other memory, map pages immediately. + if (!is_separate_heap) { + m_buffer.Map(virtual_offset, host_offset, length, perm, false); + return; + } + + { + // We are mapping part of a separate heap. + std::scoped_lock lk{m_lock}; + + auto* const map = new SeparateHeapMap{ + .vaddr = virtual_offset, + .paddr = host_offset, + .size = length, + .tick = m_tick++, + .perm = perm, + .is_resident = false, + }; + + // Insert into mappings. + m_map_count++; + m_mappings.insert(*map); + } + + // Finally, map. + this->DeferredMapSeparateHeap(virtual_offset); +} + +void HeapTracker::Unmap(size_t virtual_offset, size_t size, bool is_separate_heap) { + // If this is a separate heap... + if (is_separate_heap) { + std::scoped_lock lk{m_lock}; + + const SeparateHeapMap key{ + .vaddr = virtual_offset, + }; + + // Split at the boundaries of the region we are removing. + this->SplitHeapMapLocked(virtual_offset); + this->SplitHeapMapLocked(virtual_offset + size); + + // Erase all mappings in range. + auto it = m_mappings.find(key); + while (it != m_mappings.end() && it->vaddr < virtual_offset + size) { + // Get underlying item. + auto* const item = std::addressof(*it); + + // If resident, erase from resident map. + if (item->is_resident) { + ASSERT(--m_resident_map_count >= 0); + m_resident_mappings.erase(m_resident_mappings.iterator_to(*item)); + } + + // Erase from map. + ASSERT(--m_map_count >= 0); + it = m_mappings.erase(it); + + // Free the item. + delete item; + } + } + + // Unmap pages. + m_buffer.Unmap(virtual_offset, size, false); +} + +void HeapTracker::Protect(size_t virtual_offset, size_t size, MemoryPermission perm) { + // Ensure no rebuild occurs while reprotecting. + std::shared_lock lk{m_rebuild_lock}; + + // Split at the boundaries of the region we are reprotecting. + this->SplitHeapMap(virtual_offset, size); + + // Declare tracking variables. + const VAddr end = virtual_offset + size; + VAddr cur = virtual_offset; + + while (cur < end) { + VAddr next = cur; + bool should_protect = false; + + { + std::scoped_lock lk2{m_lock}; + + const SeparateHeapMap key{ + .vaddr = next, + }; + + // Try to get the next mapping corresponding to this address. + const auto it = m_mappings.nfind(key); + + if (it == m_mappings.end()) { + // There are no separate heap mappings remaining. + next = end; + should_protect = true; + } else if (it->vaddr == cur) { + // We are in range. + // Update permission bits. + it->perm = perm; + + // Determine next address and whether we should protect. + next = cur + it->size; + should_protect = it->is_resident; + } else /* if (it->vaddr > cur) */ { + // We weren't in range, but there is a block coming up that will be. + next = it->vaddr; + should_protect = true; + } + } + + // Clamp to end. + next = std::min(next, end); + + // Reprotect, if we need to. + if (should_protect) { + m_buffer.Protect(cur, next - cur, perm); + } + + // Advance. + cur = next; + } +} + +bool HeapTracker::DeferredMapSeparateHeap(u8* fault_address) { + if (m_buffer.IsInVirtualRange(fault_address)) { + return this->DeferredMapSeparateHeap(fault_address - m_buffer.VirtualBasePointer()); + } + + return false; +} + +bool HeapTracker::DeferredMapSeparateHeap(size_t virtual_offset) { + bool rebuild_required = false; + + { + std::scoped_lock lk{m_lock}; + + // Check to ensure this was a non-resident separate heap mapping. + const auto it = this->GetNearestHeapMapLocked(virtual_offset); + if (it == m_mappings.end() || it->is_resident) { + return false; + } + + // Update tick before possible rebuild. + it->tick = m_tick++; + + // Check if we need to rebuild. + if (m_resident_map_count > m_max_resident_map_count) { + rebuild_required = true; + } + + // Map the area. + m_buffer.Map(it->vaddr, it->paddr, it->size, it->perm, false); + + // This map is now resident. + it->is_resident = true; + m_resident_map_count++; + m_resident_mappings.insert(*it); + } + + if (rebuild_required) { + // A rebuild was required, so perform it now. + this->RebuildSeparateHeapAddressSpace(); + } + + return true; +} + +void HeapTracker::RebuildSeparateHeapAddressSpace() { + std::scoped_lock lk{m_rebuild_lock, m_lock}; + + ASSERT(!m_resident_mappings.empty()); + + // Dump half of the mappings. + // + // Despite being worse in theory, this has proven to be better in practice than more + // regularly dumping a smaller amount, because it significantly reduces average case + // lock contention. + const size_t desired_count = std::min(m_resident_map_count, m_max_resident_map_count) / 2; + const size_t evict_count = m_resident_map_count - desired_count; + auto it = m_resident_mappings.begin(); + + for (size_t i = 0; i < evict_count && it != m_resident_mappings.end(); i++) { + // Unmark and unmap. + it->is_resident = false; + m_buffer.Unmap(it->vaddr, it->size, false); + + // Advance. + ASSERT(--m_resident_map_count >= 0); + it = m_resident_mappings.erase(it); + } +} + +void HeapTracker::SplitHeapMap(VAddr offset, size_t size) { + std::scoped_lock lk{m_lock}; + + this->SplitHeapMapLocked(offset); + this->SplitHeapMapLocked(offset + size); +} + +void HeapTracker::SplitHeapMapLocked(VAddr offset) { + const auto it = this->GetNearestHeapMapLocked(offset); + if (it == m_mappings.end() || it->vaddr == offset) { + // Not contained or no split required. + return; + } + + // Cache the original values. + auto* const left = std::addressof(*it); + const size_t orig_size = left->size; + + // Adjust the left map. + const size_t left_size = offset - left->vaddr; + left->size = left_size; + + // Create the new right map. + auto* const right = new SeparateHeapMap{ + .vaddr = left->vaddr + left_size, + .paddr = left->paddr + left_size, + .size = orig_size - left_size, + .tick = left->tick, + .perm = left->perm, + .is_resident = left->is_resident, + }; + + // Insert the new right map. + m_map_count++; + m_mappings.insert(*right); + + // If resident, also insert into resident map. + if (right->is_resident) { + m_resident_map_count++; + m_resident_mappings.insert(*right); + } +} + +HeapTracker::AddrTree::iterator HeapTracker::GetNearestHeapMapLocked(VAddr offset) { + const SeparateHeapMap key{ + .vaddr = offset, + }; + + return m_mappings.find(key); +} + +} // namespace Common diff --git a/src/common/heap_tracker.h b/src/common/heap_tracker.h new file mode 100644 index 000000000..ee5b0bf43 --- /dev/null +++ b/src/common/heap_tracker.h @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include <atomic> +#include <mutex> +#include <set> +#include <shared_mutex> + +#include "common/host_memory.h" +#include "common/intrusive_red_black_tree.h" + +namespace Common { + +struct SeparateHeapMap { + Common::IntrusiveRedBlackTreeNode addr_node{}; + Common::IntrusiveRedBlackTreeNode tick_node{}; + VAddr vaddr{}; + PAddr paddr{}; + size_t size{}; + size_t tick{}; + MemoryPermission perm{}; + bool is_resident{}; +}; + +struct SeparateHeapMapAddrComparator { + static constexpr int Compare(const SeparateHeapMap& lhs, const SeparateHeapMap& rhs) { + if (lhs.vaddr < rhs.vaddr) { + return -1; + } else if (lhs.vaddr <= (rhs.vaddr + rhs.size - 1)) { + return 0; + } else { + return 1; + } + } +}; + +struct SeparateHeapMapTickComparator { + static constexpr int Compare(const SeparateHeapMap& lhs, const SeparateHeapMap& rhs) { + if (lhs.tick < rhs.tick) { + return -1; + } else if (lhs.tick > rhs.tick) { + return 1; + } else { + return SeparateHeapMapAddrComparator::Compare(lhs, rhs); + } + } +}; + +class HeapTracker { +public: + explicit HeapTracker(Common::HostMemory& buffer); + ~HeapTracker(); + + void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perm, + bool is_separate_heap); + void Unmap(size_t virtual_offset, size_t size, bool is_separate_heap); + void Protect(size_t virtual_offset, size_t length, MemoryPermission perm); + u8* VirtualBasePointer() { + return m_buffer.VirtualBasePointer(); + } + + bool DeferredMapSeparateHeap(u8* fault_address); + bool DeferredMapSeparateHeap(size_t virtual_offset); + +private: + using AddrTreeTraits = + Common::IntrusiveRedBlackTreeMemberTraitsDeferredAssert<&SeparateHeapMap::addr_node>; + using AddrTree = AddrTreeTraits::TreeType<SeparateHeapMapAddrComparator>; + + using TickTreeTraits = + Common::IntrusiveRedBlackTreeMemberTraitsDeferredAssert<&SeparateHeapMap::tick_node>; + using TickTree = TickTreeTraits::TreeType<SeparateHeapMapTickComparator>; + + AddrTree m_mappings{}; + TickTree m_resident_mappings{}; + +private: + void SplitHeapMap(VAddr offset, size_t size); + void SplitHeapMapLocked(VAddr offset); + + AddrTree::iterator GetNearestHeapMapLocked(VAddr offset); + + void RebuildSeparateHeapAddressSpace(); + +private: + Common::HostMemory& m_buffer; + const s64 m_max_resident_map_count; + + std::shared_mutex m_rebuild_lock{}; + std::mutex m_lock{}; + s64 m_map_count{}; + s64 m_resident_map_count{}; + size_t m_tick{}; +}; + +} // namespace Common diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index e540375b8..860c39e6a 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -679,7 +679,7 @@ HostMemory::HostMemory(HostMemory&&) noexcept = default; HostMemory& HostMemory::operator=(HostMemory&&) noexcept = default; void HostMemory::Map(size_t virtual_offset, size_t host_offset, size_t length, - MemoryPermission perms) { + MemoryPermission perms, bool separate_heap) { ASSERT(virtual_offset % PageAlignment == 0); ASSERT(host_offset % PageAlignment == 0); ASSERT(length % PageAlignment == 0); @@ -691,7 +691,7 @@ void HostMemory::Map(size_t virtual_offset, size_t host_offset, size_t length, impl->Map(virtual_offset + virtual_base_offset, host_offset, length, perms); } -void HostMemory::Unmap(size_t virtual_offset, size_t length) { +void HostMemory::Unmap(size_t virtual_offset, size_t length, bool separate_heap) { ASSERT(virtual_offset % PageAlignment == 0); ASSERT(length % PageAlignment == 0); ASSERT(virtual_offset + length <= virtual_size); @@ -701,14 +701,16 @@ void HostMemory::Unmap(size_t virtual_offset, size_t length) { impl->Unmap(virtual_offset + virtual_base_offset, length); } -void HostMemory::Protect(size_t virtual_offset, size_t length, bool read, bool write, - bool execute) { +void HostMemory::Protect(size_t virtual_offset, size_t length, MemoryPermission perm) { ASSERT(virtual_offset % PageAlignment == 0); ASSERT(length % PageAlignment == 0); ASSERT(virtual_offset + length <= virtual_size); if (length == 0 || !virtual_base || !impl) { return; } + const bool read = True(perm & MemoryPermission::Read); + const bool write = True(perm & MemoryPermission::Write); + const bool execute = True(perm & MemoryPermission::Execute); impl->Protect(virtual_offset + virtual_base_offset, length, read, write, execute); } diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 747c5850c..72fbb05af 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -40,11 +40,12 @@ public: HostMemory(HostMemory&& other) noexcept; HostMemory& operator=(HostMemory&& other) noexcept; - void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms); + void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms, + bool separate_heap); - void Unmap(size_t virtual_offset, size_t length); + void Unmap(size_t virtual_offset, size_t length, bool separate_heap); - void Protect(size_t virtual_offset, size_t length, bool read, bool write, bool execute = false); + void Protect(size_t virtual_offset, size_t length, MemoryPermission perms); void EnableDirectMappedAddress(); @@ -64,6 +65,10 @@ public: return virtual_base; } + bool IsInVirtualRange(void* address) const noexcept { + return address >= virtual_base && address < virtual_base + virtual_size; + } + private: size_t backing_size{}; size_t virtual_size{}; diff --git a/src/common/logging/backend.cpp b/src/common/logging/backend.cpp index d4f27197c..7a267f8c0 100644 --- a/src/common/logging/backend.cpp +++ b/src/common/logging/backend.cpp @@ -208,6 +208,10 @@ public: instance->StartBackendThread(); } + static void Stop() { + instance->StopBackendThread(); + } + Impl(const Impl&) = delete; Impl& operator=(const Impl&) = delete; @@ -259,6 +263,15 @@ private: }); } + void StopBackendThread() { + backend_thread.request_stop(); + if (backend_thread.joinable()) { + backend_thread.join(); + } + + ForEachBackend([](Backend& backend) { backend.Flush(); }); + } + Entry CreateEntry(Class log_class, Level log_level, const char* filename, unsigned int line_nr, const char* function, std::string&& message) const { using std::chrono::duration_cast; @@ -313,6 +326,10 @@ void Start() { Impl::Start(); } +void Stop() { + Impl::Stop(); +} + void DisableLoggingInTests() { initialization_in_progress_suppress_logging = true; } diff --git a/src/common/logging/backend.h b/src/common/logging/backend.h index 12e5e2498..2a9926e9e 100644 --- a/src/common/logging/backend.h +++ b/src/common/logging/backend.h @@ -14,6 +14,9 @@ void Initialize(); void Start(); +/// Explicitly stops the logger thread and flushes the buffers +void Stop(); + void DisableLoggingInTests(); /** diff --git a/src/common/ring_buffer.h b/src/common/ring_buffer.h index 5c961b202..e7e9fdb38 100644 --- a/src/common/ring_buffer.h +++ b/src/common/ring_buffer.h @@ -103,7 +103,7 @@ private: // Having them on the same cache-line would result in false-sharing between them. // TODO: Remove this ifdef whenever clang and GCC support // std::hardware_destructive_interference_size. -#if defined(_MSC_VER) && _MSC_VER >= 1911 +#ifdef __cpp_lib_hardware_interference_size alignas(std::hardware_destructive_interference_size) std::atomic_size_t m_read_index{0}; alignas(std::hardware_destructive_interference_size) std::atomic_size_t m_write_index{0}; #else diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 96ab39cb8..367d01dc7 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -978,6 +978,7 @@ endif() if (ARCHITECTURE_x86_64 OR ARCHITECTURE_arm64) target_sources(core PRIVATE + arm/dynarmic/arm_dynarmic.cpp arm/dynarmic/arm_dynarmic.h arm/dynarmic/arm_dynarmic_64.cpp arm/dynarmic/arm_dynarmic_64.h @@ -987,6 +988,8 @@ if (ARCHITECTURE_x86_64 OR ARCHITECTURE_arm64) arm/dynarmic/dynarmic_cp15.h arm/dynarmic/dynarmic_exclusive_monitor.cpp arm/dynarmic/dynarmic_exclusive_monitor.h + hle/service/jit/jit_code_memory.cpp + hle/service/jit/jit_code_memory.h hle/service/jit/jit_context.cpp hle/service/jit/jit_context.h hle/service/jit/jit.cpp diff --git a/src/core/arm/arm_interface.cpp b/src/core/arm/arm_interface.cpp index 698c9c8ad..5dc7e5d59 100644 --- a/src/core/arm/arm_interface.cpp +++ b/src/core/arm/arm_interface.cpp @@ -9,7 +9,7 @@ namespace Core { -void ArmInterface::LogBacktrace(const Kernel::KProcess* process) const { +void ArmInterface::LogBacktrace(Kernel::KProcess* process) const { Kernel::Svc::ThreadContext ctx; this->GetContext(ctx); diff --git a/src/core/arm/arm_interface.h b/src/core/arm/arm_interface.h index 806c7c9e9..495963eef 100644 --- a/src/core/arm/arm_interface.h +++ b/src/core/arm/arm_interface.h @@ -95,7 +95,7 @@ public: virtual void SignalInterrupt(Kernel::KThread* thread) = 0; // Stack trace generation. - void LogBacktrace(const Kernel::KProcess* process) const; + void LogBacktrace(Kernel::KProcess* process) const; // Debug functionality. virtual const Kernel::DebugWatchpoint* HaltedWatchpoint() const = 0; diff --git a/src/core/arm/debug.cpp b/src/core/arm/debug.cpp index af1c34bc3..854509463 100644 --- a/src/core/arm/debug.cpp +++ b/src/core/arm/debug.cpp @@ -79,7 +79,7 @@ constexpr std::array<u64, 2> SegmentBases{ 0x7100000000ULL, }; -void SymbolicateBacktrace(const Kernel::KProcess* process, std::vector<BacktraceEntry>& out) { +void SymbolicateBacktrace(Kernel::KProcess* process, std::vector<BacktraceEntry>& out) { auto modules = FindModules(process); const bool is_64 = process->Is64Bit(); @@ -118,7 +118,7 @@ void SymbolicateBacktrace(const Kernel::KProcess* process, std::vector<Backtrace } } -std::vector<BacktraceEntry> GetAArch64Backtrace(const Kernel::KProcess* process, +std::vector<BacktraceEntry> GetAArch64Backtrace(Kernel::KProcess* process, const Kernel::Svc::ThreadContext& ctx) { std::vector<BacktraceEntry> out; auto& memory = process->GetMemory(); @@ -144,7 +144,7 @@ std::vector<BacktraceEntry> GetAArch64Backtrace(const Kernel::KProcess* process, return out; } -std::vector<BacktraceEntry> GetAArch32Backtrace(const Kernel::KProcess* process, +std::vector<BacktraceEntry> GetAArch32Backtrace(Kernel::KProcess* process, const Kernel::Svc::ThreadContext& ctx) { std::vector<BacktraceEntry> out; auto& memory = process->GetMemory(); @@ -173,7 +173,7 @@ std::vector<BacktraceEntry> GetAArch32Backtrace(const Kernel::KProcess* process, } // namespace std::optional<std::string> GetThreadName(const Kernel::KThread* thread) { - const auto* process = thread->GetOwnerProcess(); + auto* process = thread->GetOwnerProcess(); if (process->Is64Bit()) { return GetNameFromThreadType64(process->GetMemory(), *thread); } else { @@ -248,7 +248,7 @@ Kernel::KProcessAddress GetModuleEnd(const Kernel::KProcess* process, return cur_addr - 1; } -Loader::AppLoader::Modules FindModules(const Kernel::KProcess* process) { +Loader::AppLoader::Modules FindModules(Kernel::KProcess* process) { Loader::AppLoader::Modules modules; auto& page_table = process->GetPageTable(); @@ -312,7 +312,7 @@ Loader::AppLoader::Modules FindModules(const Kernel::KProcess* process) { return modules; } -Kernel::KProcessAddress FindMainModuleEntrypoint(const Kernel::KProcess* process) { +Kernel::KProcessAddress FindMainModuleEntrypoint(Kernel::KProcess* process) { // Do we have any loaded executable sections? auto modules = FindModules(process); @@ -337,7 +337,7 @@ void InvalidateInstructionCacheRange(const Kernel::KProcess* process, u64 addres } } -std::vector<BacktraceEntry> GetBacktraceFromContext(const Kernel::KProcess* process, +std::vector<BacktraceEntry> GetBacktraceFromContext(Kernel::KProcess* process, const Kernel::Svc::ThreadContext& ctx) { if (process->Is64Bit()) { return GetAArch64Backtrace(process, ctx); diff --git a/src/core/arm/debug.h b/src/core/arm/debug.h index c542633db..3cd671365 100644 --- a/src/core/arm/debug.h +++ b/src/core/arm/debug.h @@ -14,9 +14,9 @@ std::optional<std::string> GetThreadName(const Kernel::KThread* thread); std::string_view GetThreadWaitReason(const Kernel::KThread* thread); std::string GetThreadState(const Kernel::KThread* thread); -Loader::AppLoader::Modules FindModules(const Kernel::KProcess* process); +Loader::AppLoader::Modules FindModules(Kernel::KProcess* process); Kernel::KProcessAddress GetModuleEnd(const Kernel::KProcess* process, Kernel::KProcessAddress base); -Kernel::KProcessAddress FindMainModuleEntrypoint(const Kernel::KProcess* process); +Kernel::KProcessAddress FindMainModuleEntrypoint(Kernel::KProcess* process); void InvalidateInstructionCacheRange(const Kernel::KProcess* process, u64 address, u64 size); @@ -28,7 +28,7 @@ struct BacktraceEntry { std::string name; }; -std::vector<BacktraceEntry> GetBacktraceFromContext(const Kernel::KProcess* process, +std::vector<BacktraceEntry> GetBacktraceFromContext(Kernel::KProcess* process, const Kernel::Svc::ThreadContext& ctx); std::vector<BacktraceEntry> GetBacktrace(const Kernel::KThread* thread); diff --git a/src/core/arm/dynarmic/arm_dynarmic.cpp b/src/core/arm/dynarmic/arm_dynarmic.cpp new file mode 100644 index 000000000..e6e9fc45b --- /dev/null +++ b/src/core/arm/dynarmic/arm_dynarmic.cpp @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#ifdef __linux__ + +#include "common/signal_chain.h" + +#include "core/arm/dynarmic/arm_dynarmic.h" +#include "core/hle/kernel/k_process.h" +#include "core/memory.h" + +namespace Core { + +namespace { + +thread_local Core::Memory::Memory* g_current_memory{}; +std::once_flag g_registered{}; +struct sigaction g_old_segv {}; + +void HandleSigSegv(int sig, siginfo_t* info, void* ctx) { + if (g_current_memory && g_current_memory->InvalidateSeparateHeap(info->si_addr)) { + return; + } + + return g_old_segv.sa_sigaction(sig, info, ctx); +} + +} // namespace + +ScopedJitExecution::ScopedJitExecution(Kernel::KProcess* process) { + g_current_memory = std::addressof(process->GetMemory()); +} + +ScopedJitExecution::~ScopedJitExecution() { + g_current_memory = nullptr; +} + +void ScopedJitExecution::RegisterHandler() { + std::call_once(g_registered, [] { + struct sigaction sa {}; + sa.sa_sigaction = &HandleSigSegv; + sa.sa_flags = SA_SIGINFO | SA_ONSTACK; + Common::SigAction(SIGSEGV, std::addressof(sa), std::addressof(g_old_segv)); + }); +} + +} // namespace Core + +#endif diff --git a/src/core/arm/dynarmic/arm_dynarmic.h b/src/core/arm/dynarmic/arm_dynarmic.h index eef7c3116..53dd18815 100644 --- a/src/core/arm/dynarmic/arm_dynarmic.h +++ b/src/core/arm/dynarmic/arm_dynarmic.h @@ -26,4 +26,24 @@ constexpr HaltReason TranslateHaltReason(Dynarmic::HaltReason hr) { return static_cast<HaltReason>(hr); } +#ifdef __linux__ + +class ScopedJitExecution { +public: + explicit ScopedJitExecution(Kernel::KProcess* process); + ~ScopedJitExecution(); + static void RegisterHandler(); +}; + +#else + +class ScopedJitExecution { +public: + explicit ScopedJitExecution(Kernel::KProcess* process) {} + ~ScopedJitExecution() {} + static void RegisterHandler() {} +}; + +#endif + } // namespace Core diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.cpp b/src/core/arm/dynarmic/arm_dynarmic_32.cpp index f34865e26..36478f722 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_32.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_32.cpp @@ -15,7 +15,7 @@ using namespace Common::Literals; class DynarmicCallbacks32 : public Dynarmic::A32::UserCallbacks { public: - explicit DynarmicCallbacks32(ArmDynarmic32& parent, const Kernel::KProcess* process) + explicit DynarmicCallbacks32(ArmDynarmic32& parent, Kernel::KProcess* process) : m_parent{parent}, m_memory(process->GetMemory()), m_process(process), m_debugger_enabled{parent.m_system.DebuggerEnabled()}, m_check_memory_access{m_debugger_enabled || @@ -169,7 +169,7 @@ public: ArmDynarmic32& m_parent; Core::Memory::Memory& m_memory; - const Kernel::KProcess* m_process{}; + Kernel::KProcess* m_process{}; const bool m_debugger_enabled{}; const bool m_check_memory_access{}; static constexpr u64 MinimumRunCycles = 10000U; @@ -331,11 +331,15 @@ bool ArmDynarmic32::IsInThumbMode() const { } HaltReason ArmDynarmic32::RunThread(Kernel::KThread* thread) { + ScopedJitExecution sj(thread->GetOwnerProcess()); + m_jit->ClearExclusiveState(); return TranslateHaltReason(m_jit->Run()); } HaltReason ArmDynarmic32::StepThread(Kernel::KThread* thread) { + ScopedJitExecution sj(thread->GetOwnerProcess()); + m_jit->ClearExclusiveState(); return TranslateHaltReason(m_jit->Step()); } @@ -370,13 +374,14 @@ void ArmDynarmic32::RewindBreakpointInstruction() { this->SetContext(m_breakpoint_context); } -ArmDynarmic32::ArmDynarmic32(System& system, bool uses_wall_clock, const Kernel::KProcess* process, +ArmDynarmic32::ArmDynarmic32(System& system, bool uses_wall_clock, Kernel::KProcess* process, DynarmicExclusiveMonitor& exclusive_monitor, std::size_t core_index) : ArmInterface{uses_wall_clock}, m_system{system}, m_exclusive_monitor{exclusive_monitor}, m_cb(std::make_unique<DynarmicCallbacks32>(*this, process)), m_cp15(std::make_shared<DynarmicCP15>(*this)), m_core_index{core_index} { auto& page_table_impl = process->GetPageTable().GetBasePageTable().GetImpl(); m_jit = MakeJit(&page_table_impl); + ScopedJitExecution::RegisterHandler(); } ArmDynarmic32::~ArmDynarmic32() = default; diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.h b/src/core/arm/dynarmic/arm_dynarmic_32.h index 185ac7cbf..b580efe61 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_32.h +++ b/src/core/arm/dynarmic/arm_dynarmic_32.h @@ -20,7 +20,7 @@ class System; class ArmDynarmic32 final : public ArmInterface { public: - ArmDynarmic32(System& system, bool uses_wall_clock, const Kernel::KProcess* process, + ArmDynarmic32(System& system, bool uses_wall_clock, Kernel::KProcess* process, DynarmicExclusiveMonitor& exclusive_monitor, std::size_t core_index); ~ArmDynarmic32() override; diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index dff14756e..c811c8ad5 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -15,7 +15,7 @@ using namespace Common::Literals; class DynarmicCallbacks64 : public Dynarmic::A64::UserCallbacks { public: - explicit DynarmicCallbacks64(ArmDynarmic64& parent, const Kernel::KProcess* process) + explicit DynarmicCallbacks64(ArmDynarmic64& parent, Kernel::KProcess* process) : m_parent{parent}, m_memory(process->GetMemory()), m_process(process), m_debugger_enabled{parent.m_system.DebuggerEnabled()}, m_check_memory_access{m_debugger_enabled || @@ -216,7 +216,7 @@ public: Core::Memory::Memory& m_memory; u64 m_tpidrro_el0{}; u64 m_tpidr_el0{}; - const Kernel::KProcess* m_process{}; + Kernel::KProcess* m_process{}; const bool m_debugger_enabled{}; const bool m_check_memory_access{}; static constexpr u64 MinimumRunCycles = 10000U; @@ -362,11 +362,15 @@ std::shared_ptr<Dynarmic::A64::Jit> ArmDynarmic64::MakeJit(Common::PageTable* pa } HaltReason ArmDynarmic64::RunThread(Kernel::KThread* thread) { + ScopedJitExecution sj(thread->GetOwnerProcess()); + m_jit->ClearExclusiveState(); return TranslateHaltReason(m_jit->Run()); } HaltReason ArmDynarmic64::StepThread(Kernel::KThread* thread) { + ScopedJitExecution sj(thread->GetOwnerProcess()); + m_jit->ClearExclusiveState(); return TranslateHaltReason(m_jit->Step()); } @@ -399,13 +403,14 @@ void ArmDynarmic64::RewindBreakpointInstruction() { this->SetContext(m_breakpoint_context); } -ArmDynarmic64::ArmDynarmic64(System& system, bool uses_wall_clock, const Kernel::KProcess* process, +ArmDynarmic64::ArmDynarmic64(System& system, bool uses_wall_clock, Kernel::KProcess* process, DynarmicExclusiveMonitor& exclusive_monitor, std::size_t core_index) : ArmInterface{uses_wall_clock}, m_system{system}, m_exclusive_monitor{exclusive_monitor}, m_cb(std::make_unique<DynarmicCallbacks64>(*this, process)), m_core_index{core_index} { auto& page_table = process->GetPageTable().GetBasePageTable(); auto& page_table_impl = page_table.GetImpl(); m_jit = MakeJit(&page_table_impl, page_table.GetAddressSpaceWidth()); + ScopedJitExecution::RegisterHandler(); } ArmDynarmic64::~ArmDynarmic64() = default; diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.h b/src/core/arm/dynarmic/arm_dynarmic_64.h index 4f3dd026f..08cd982b3 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.h +++ b/src/core/arm/dynarmic/arm_dynarmic_64.h @@ -25,7 +25,7 @@ class System; class ArmDynarmic64 final : public ArmInterface { public: - ArmDynarmic64(System& system, bool uses_wall_clock, const Kernel::KProcess* process, + ArmDynarmic64(System& system, bool uses_wall_clock, Kernel::KProcess* process, DynarmicExclusiveMonitor& exclusive_monitor, std::size_t core_index); ~ArmDynarmic64() override; diff --git a/src/core/core.cpp b/src/core/core.cpp index b14f74976..66f444d39 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -28,7 +28,6 @@ #include "core/file_sys/savedata_factory.h" #include "core/file_sys/vfs_concat.h" #include "core/file_sys/vfs_real.h" -#include "core/gpu_dirty_memory_manager.h" #include "core/hid/hid_core.h" #include "core/hle/kernel/k_memory_manager.h" #include "core/hle/kernel/k_process.h" @@ -130,11 +129,8 @@ FileSys::VirtualFile GetGameFileFromPath(const FileSys::VirtualFilesystem& vfs, struct System::Impl { explicit Impl(System& system) - : kernel{system}, fs_controller{system}, memory{system}, hid_core{}, room_network{}, - cpu_manager{system}, reporter{system}, applet_manager{system}, profile_manager{}, - time_manager{system}, gpu_dirty_memory_write_manager{} { - memory.SetGPUDirtyManagers(gpu_dirty_memory_write_manager); - } + : kernel{system}, fs_controller{system}, hid_core{}, room_network{}, cpu_manager{system}, + reporter{system}, applet_manager{system}, profile_manager{}, time_manager{system} {} void Initialize(System& system) { device_memory = std::make_unique<Core::DeviceMemory>(); @@ -241,17 +237,17 @@ struct System::Impl { debugger = std::make_unique<Debugger>(system, port); } - SystemResultStatus SetupForApplicationProcess(System& system, Frontend::EmuWindow& emu_window) { + void InitializeKernel(System& system) { LOG_DEBUG(Core, "initialized OK"); // Setting changes may require a full system reinitialization (e.g., disabling multicore). ReinitializeIfNecessary(system); - memory.SetGPUDirtyManagers(gpu_dirty_memory_write_manager); - kernel.Initialize(); cpu_manager.Initialize(); + } + SystemResultStatus SetupForApplicationProcess(System& system, Frontend::EmuWindow& emu_window) { /// Reset all glue registrations arp_manager.ResetAll(); @@ -300,17 +296,9 @@ struct System::Impl { return SystemResultStatus::ErrorGetLoader; } - SystemResultStatus init_result{SetupForApplicationProcess(system, emu_window)}; - if (init_result != SystemResultStatus::Success) { - LOG_CRITICAL(Core, "Failed to initialize system (Error {})!", - static_cast<int>(init_result)); - ShutdownMainProcess(); - return init_result; - } - - telemetry_session->AddInitialInfo(*app_loader, fs_controller, *content_provider); + InitializeKernel(system); - // Create the process. + // Create the application process. auto main_process = Kernel::KProcess::Create(system.Kernel()); Kernel::KProcess::Register(system.Kernel(), main_process); kernel.AppendNewProcess(main_process); @@ -323,7 +311,18 @@ struct System::Impl { return static_cast<SystemResultStatus>( static_cast<u32>(SystemResultStatus::ErrorLoader) + static_cast<u32>(load_result)); } + + // Set up the rest of the system. + SystemResultStatus init_result{SetupForApplicationProcess(system, emu_window)}; + if (init_result != SystemResultStatus::Success) { + LOG_CRITICAL(Core, "Failed to initialize system (Error {})!", + static_cast<int>(init_result)); + ShutdownMainProcess(); + return init_result; + } + AddGlueRegistrationForProcess(*app_loader, *main_process); + telemetry_session->AddInitialInfo(*app_loader, fs_controller, *content_provider); // Initialize cheat engine if (cheat_engine) { @@ -426,7 +425,6 @@ struct System::Impl { cpu_manager.Shutdown(); debugger.reset(); kernel.Shutdown(); - memory.Reset(); Network::RestartSocketOperations(); if (auto room_member = room_network.GetRoomMember().lock()) { @@ -507,7 +505,6 @@ struct System::Impl { std::unique_ptr<Tegra::Host1x::Host1x> host1x_core; std::unique_ptr<Core::DeviceMemory> device_memory; std::unique_ptr<AudioCore::AudioCore> audio_core; - Core::Memory::Memory memory; Core::HID::HIDCore hid_core; Network::RoomNetwork room_network; @@ -567,9 +564,6 @@ struct System::Impl { std::array<u64, Core::Hardware::NUM_CPU_CORES> dynarmic_ticks{}; std::array<MicroProfileToken, Core::Hardware::NUM_CPU_CORES> microprofile_cpu{}; - std::array<Core::GPUDirtyMemoryManager, Core::Hardware::NUM_CPU_CORES> - gpu_dirty_memory_write_manager{}; - std::deque<std::vector<u8>> user_channel; }; @@ -652,29 +646,12 @@ void System::PrepareReschedule(const u32 core_index) { impl->kernel.PrepareReschedule(core_index); } -Core::GPUDirtyMemoryManager& System::CurrentGPUDirtyMemoryManager() { - const std::size_t core = impl->kernel.GetCurrentHostThreadID(); - return impl->gpu_dirty_memory_write_manager[core < Core::Hardware::NUM_CPU_CORES - ? core - : Core::Hardware::NUM_CPU_CORES - 1]; -} - -/// Provides a constant reference to the current gou dirty memory manager. -const Core::GPUDirtyMemoryManager& System::CurrentGPUDirtyMemoryManager() const { - const std::size_t core = impl->kernel.GetCurrentHostThreadID(); - return impl->gpu_dirty_memory_write_manager[core < Core::Hardware::NUM_CPU_CORES - ? core - : Core::Hardware::NUM_CPU_CORES - 1]; -} - size_t System::GetCurrentHostThreadID() const { return impl->kernel.GetCurrentHostThreadID(); } void System::GatherGPUDirtyMemory(std::function<void(VAddr, size_t)>& callback) { - for (auto& manager : impl->gpu_dirty_memory_write_manager) { - manager.Gather(callback); - } + return this->ApplicationProcess()->GatherGPUDirtyMemory(callback); } PerfStatsResults System::GetAndResetPerfStats() { @@ -723,20 +700,12 @@ const Kernel::KProcess* System::ApplicationProcess() const { return impl->kernel.ApplicationProcess(); } -ExclusiveMonitor& System::Monitor() { - return impl->kernel.GetExclusiveMonitor(); -} - -const ExclusiveMonitor& System::Monitor() const { - return impl->kernel.GetExclusiveMonitor(); -} - Memory::Memory& System::ApplicationMemory() { - return impl->memory; + return impl->kernel.ApplicationProcess()->GetMemory(); } const Core::Memory::Memory& System::ApplicationMemory() const { - return impl->memory; + return impl->kernel.ApplicationProcess()->GetMemory(); } Tegra::GPU& System::GPU() { diff --git a/src/core/core.h b/src/core/core.h index 473204db7..ba5add0dc 100644 --- a/src/core/core.h +++ b/src/core/core.h @@ -116,7 +116,6 @@ class CpuManager; class Debugger; class DeviceMemory; class ExclusiveMonitor; -class GPUDirtyMemoryManager; class PerfStats; class Reporter; class SpeedLimiter; @@ -225,12 +224,6 @@ public: /// Prepare the core emulation for a reschedule void PrepareReschedule(u32 core_index); - /// Provides a reference to the gou dirty memory manager. - [[nodiscard]] Core::GPUDirtyMemoryManager& CurrentGPUDirtyMemoryManager(); - - /// Provides a constant reference to the current gou dirty memory manager. - [[nodiscard]] const Core::GPUDirtyMemoryManager& CurrentGPUDirtyMemoryManager() const; - void GatherGPUDirtyMemory(std::function<void(VAddr, size_t)>& callback); [[nodiscard]] size_t GetCurrentHostThreadID() const; @@ -250,12 +243,6 @@ public: /// Gets a const reference to the underlying CPU manager [[nodiscard]] const CpuManager& GetCpuManager() const; - /// Gets a reference to the exclusive monitor - [[nodiscard]] ExclusiveMonitor& Monitor(); - - /// Gets a constant reference to the exclusive monitor - [[nodiscard]] const ExclusiveMonitor& Monitor() const; - /// Gets a mutable reference to the system memory instance. [[nodiscard]] Core::Memory::Memory& ApplicationMemory(); diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cpp index 7be1322cc..31033634c 100644 --- a/src/core/file_sys/ips_layer.cpp +++ b/src/core/file_sys/ips_layer.cpp @@ -73,6 +73,9 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { return nullptr; auto in_data = in->ReadAllBytes(); + if (in_data.size() == 0) { + return nullptr; + } std::vector<u8> temp(type == IPSFileType::IPS ? 3 : 4); u64 offset = 5; // After header @@ -88,6 +91,10 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { else real_offset = (temp[0] << 16) | (temp[1] << 8) | temp[2]; + if (real_offset > in_data.size()) { + return nullptr; + } + u16 data_size{}; if (ips->ReadObject(&data_size, offset) != sizeof(u16)) return nullptr; diff --git a/src/core/file_sys/program_metadata.cpp b/src/core/file_sys/program_metadata.cpp index 763a44fee..539c7f7af 100644 --- a/src/core/file_sys/program_metadata.cpp +++ b/src/core/file_sys/program_metadata.cpp @@ -166,6 +166,10 @@ u32 ProgramMetadata::GetSystemResourceSize() const { return npdm_header.system_resource_size; } +PoolPartition ProgramMetadata::GetPoolPartition() const { + return acid_header.pool_partition; +} + const ProgramMetadata::KernelCapabilityDescriptors& ProgramMetadata::GetKernelCapabilities() const { return aci_kernel_capabilities; } @@ -201,7 +205,7 @@ void ProgramMetadata::Print() const { // Begin ACID printing (potential perms, signed) LOG_DEBUG(Service_FS, "Magic: {:.4}", acid_header.magic.data()); LOG_DEBUG(Service_FS, "Flags: 0x{:02X}", acid_header.flags); - LOG_DEBUG(Service_FS, " > Is Retail: {}", acid_header.is_retail ? "YES" : "NO"); + LOG_DEBUG(Service_FS, " > Is Retail: {}", acid_header.production_flag ? "YES" : "NO"); LOG_DEBUG(Service_FS, "Title ID Min: 0x{:016X}", acid_header.title_id_min); LOG_DEBUG(Service_FS, "Title ID Max: 0x{:016X}", acid_header.title_id_max); LOG_DEBUG(Service_FS, "Filesystem Access: 0x{:016X}\n", acid_file_access.permissions); diff --git a/src/core/file_sys/program_metadata.h b/src/core/file_sys/program_metadata.h index 76ee97d78..a53092b87 100644 --- a/src/core/file_sys/program_metadata.h +++ b/src/core/file_sys/program_metadata.h @@ -34,6 +34,13 @@ enum class ProgramFilePermission : u64 { Everything = 1ULL << 63, }; +enum class PoolPartition : u32 { + Application = 0, + Applet = 1, + System = 2, + SystemNonSecure = 3, +}; + /** * Helper which implements an interface to parse Program Description Metadata (NPDM) * Data can either be loaded from a file path or with data and an offset into it. @@ -72,6 +79,7 @@ public: u64 GetTitleID() const; u64 GetFilesystemPermissions() const; u32 GetSystemResourceSize() const; + PoolPartition GetPoolPartition() const; const KernelCapabilityDescriptors& GetKernelCapabilities() const; const std::array<u8, 0x10>& GetName() const { return npdm_header.application_name; @@ -116,8 +124,9 @@ private: union { u32 flags; - BitField<0, 1, u32> is_retail; - BitField<1, 31, u32> flags_unk; + BitField<0, 1, u32> production_flag; + BitField<1, 1, u32> unqualified_approval; + BitField<2, 4, PoolPartition> pool_partition; }; u64_le title_id_min; u64_le title_id_max; diff --git a/src/core/hle/kernel/k_address_arbiter.cpp b/src/core/hle/kernel/k_address_arbiter.cpp index 78d43d729..48889253d 100644 --- a/src/core/hle/kernel/k_address_arbiter.cpp +++ b/src/core/hle/kernel/k_address_arbiter.cpp @@ -4,6 +4,7 @@ #include "core/arm/exclusive_monitor.h" #include "core/core.h" #include "core/hle/kernel/k_address_arbiter.h" +#include "core/hle/kernel/k_process.h" #include "core/hle/kernel/k_scheduler.h" #include "core/hle/kernel/k_scoped_scheduler_lock_and_sleep.h" #include "core/hle/kernel/k_thread.h" @@ -26,9 +27,9 @@ bool ReadFromUser(KernelCore& kernel, s32* out, KProcessAddress address) { return true; } -bool DecrementIfLessThan(Core::System& system, s32* out, KProcessAddress address, s32 value) { - auto& monitor = system.Monitor(); - const auto current_core = system.Kernel().CurrentPhysicalCoreIndex(); +bool DecrementIfLessThan(KernelCore& kernel, s32* out, KProcessAddress address, s32 value) { + auto& monitor = GetCurrentProcess(kernel).GetExclusiveMonitor(); + const auto current_core = kernel.CurrentPhysicalCoreIndex(); // NOTE: If scheduler lock is not held here, interrupt disable is required. // KScopedInterruptDisable di; @@ -66,10 +67,10 @@ bool DecrementIfLessThan(Core::System& system, s32* out, KProcessAddress address return true; } -bool UpdateIfEqual(Core::System& system, s32* out, KProcessAddress address, s32 value, +bool UpdateIfEqual(KernelCore& kernel, s32* out, KProcessAddress address, s32 value, s32 new_value) { - auto& monitor = system.Monitor(); - const auto current_core = system.Kernel().CurrentPhysicalCoreIndex(); + auto& monitor = GetCurrentProcess(kernel).GetExclusiveMonitor(); + const auto current_core = kernel.CurrentPhysicalCoreIndex(); // NOTE: If scheduler lock is not held here, interrupt disable is required. // KScopedInterruptDisable di; @@ -159,7 +160,7 @@ Result KAddressArbiter::SignalAndIncrementIfEqual(uint64_t addr, s32 value, s32 // Check the userspace value. s32 user_value{}; - R_UNLESS(UpdateIfEqual(m_system, std::addressof(user_value), addr, value, value + 1), + R_UNLESS(UpdateIfEqual(m_kernel, std::addressof(user_value), addr, value, value + 1), ResultInvalidCurrentMemory); R_UNLESS(user_value == value, ResultInvalidState); @@ -219,7 +220,7 @@ Result KAddressArbiter::SignalAndModifyByWaitingCountIfEqual(uint64_t addr, s32 s32 user_value{}; bool succeeded{}; if (value != new_value) { - succeeded = UpdateIfEqual(m_system, std::addressof(user_value), addr, value, new_value); + succeeded = UpdateIfEqual(m_kernel, std::addressof(user_value), addr, value, new_value); } else { succeeded = ReadFromUser(m_kernel, std::addressof(user_value), addr); } @@ -262,7 +263,7 @@ Result KAddressArbiter::WaitIfLessThan(uint64_t addr, s32 value, bool decrement, s32 user_value{}; bool succeeded{}; if (decrement) { - succeeded = DecrementIfLessThan(m_system, std::addressof(user_value), addr, value); + succeeded = DecrementIfLessThan(m_kernel, std::addressof(user_value), addr, value); } else { succeeded = ReadFromUser(m_kernel, std::addressof(user_value), addr); } diff --git a/src/core/hle/kernel/k_auto_object_container.cpp b/src/core/hle/kernel/k_auto_object_container.cpp index 636b3f993..7bea1a1c2 100644 --- a/src/core/hle/kernel/k_auto_object_container.cpp +++ b/src/core/hle/kernel/k_auto_object_container.cpp @@ -8,19 +8,22 @@ namespace Kernel { void KAutoObjectWithListContainer::Register(KAutoObjectWithList* obj) { - KScopedLightLock lk(m_lock); + // KScopedInterruptDisable di; + KScopedSpinLock lk(m_lock); m_object_list.insert_unique(*obj); } void KAutoObjectWithListContainer::Unregister(KAutoObjectWithList* obj) { - KScopedLightLock lk(m_lock); + // KScopedInterruptDisable di; + KScopedSpinLock lk(m_lock); m_object_list.erase(*obj); } size_t KAutoObjectWithListContainer::GetOwnedCount(KProcess* owner) { - KScopedLightLock lk(m_lock); + // KScopedInterruptDisable di; + KScopedSpinLock lk(m_lock); return std::count_if(m_object_list.begin(), m_object_list.end(), [&](const auto& obj) { return obj.GetOwner() == owner; }); diff --git a/src/core/hle/kernel/k_auto_object_container.h b/src/core/hle/kernel/k_auto_object_container.h index badd75d2a..770743d21 100644 --- a/src/core/hle/kernel/k_auto_object_container.h +++ b/src/core/hle/kernel/k_auto_object_container.h @@ -7,7 +7,7 @@ #include "common/common_funcs.h" #include "core/hle/kernel/k_auto_object.h" -#include "core/hle/kernel/k_light_lock.h" +#include "core/hle/kernel/k_spin_lock.h" namespace Kernel { @@ -21,32 +21,7 @@ public: using ListType = boost::intrusive::rbtree<KAutoObjectWithList>; - class ListAccessor : public KScopedLightLock { - public: - explicit ListAccessor(KAutoObjectWithListContainer* container) - : KScopedLightLock(container->m_lock), m_list(container->m_object_list) {} - explicit ListAccessor(KAutoObjectWithListContainer& container) - : KScopedLightLock(container.m_lock), m_list(container.m_object_list) {} - - typename ListType::iterator begin() const { - return m_list.begin(); - } - - typename ListType::iterator end() const { - return m_list.end(); - } - - typename ListType::iterator find(typename ListType::const_reference ref) const { - return m_list.find(ref); - } - - private: - ListType& m_list; - }; - - friend class ListAccessor; - - KAutoObjectWithListContainer(KernelCore& kernel) : m_lock(kernel), m_object_list() {} + KAutoObjectWithListContainer(KernelCore& kernel) : m_lock(), m_object_list() {} void Initialize() {} void Finalize() {} @@ -56,7 +31,7 @@ public: size_t GetOwnedCount(KProcess* owner); private: - KLightLock m_lock; + KSpinLock m_lock; ListType m_object_list; }; diff --git a/src/core/hle/kernel/k_capabilities.cpp b/src/core/hle/kernel/k_capabilities.cpp index 274fee493..d2288c30d 100644 --- a/src/core/hle/kernel/k_capabilities.cpp +++ b/src/core/hle/kernel/k_capabilities.cpp @@ -185,6 +185,10 @@ Result KCapabilities::ProcessMapRegionCapability(const u32 cap, F f) { case RegionType::NoMapping: break; case RegionType::KernelTraceBuffer: + if constexpr (!IsKTraceEnabled) { + break; + } + [[fallthrough]]; case RegionType::OnMemoryBootImage: case RegionType::DTB: R_TRY(f(MemoryRegions[static_cast<u32>(type)], perm)); @@ -330,8 +334,6 @@ Result KCapabilities::SetCapabilities(std::span<const u32> caps, KProcessPageTab // Map the range. R_TRY(this->MapRange_(cap, size_cap, page_table)); - } else if (GetCapabilityType(cap) == CapabilityType::MapRegion && !IsKTraceEnabled) { - continue; } else { R_TRY(this->SetCapability(cap, set_flags, set_svc, page_table)); } diff --git a/src/core/hle/kernel/k_client_port.cpp b/src/core/hle/kernel/k_client_port.cpp index 11b1b977e..68cea978a 100644 --- a/src/core/hle/kernel/k_client_port.cpp +++ b/src/core/hle/kernel/k_client_port.cpp @@ -58,9 +58,8 @@ Result KClientPort::CreateSession(KClientSession** out) { KSession* session{}; // Reserve a new session from the resource limit. - //! FIXME: we are reserving this from the wrong resource limit! - KScopedResourceReservation session_reservation( - m_kernel.ApplicationProcess()->GetResourceLimit(), LimitableResource::SessionCountMax); + KScopedResourceReservation session_reservation(GetCurrentProcessPointer(m_kernel), + LimitableResource::SessionCountMax); R_UNLESS(session_reservation.Succeeded(), ResultLimitReached); // Allocate a session normally. diff --git a/src/core/hle/kernel/k_condition_variable.cpp b/src/core/hle/kernel/k_condition_variable.cpp index 7633a51fb..94ea3527a 100644 --- a/src/core/hle/kernel/k_condition_variable.cpp +++ b/src/core/hle/kernel/k_condition_variable.cpp @@ -28,10 +28,10 @@ bool WriteToUser(KernelCore& kernel, KProcessAddress address, const u32* p) { return true; } -bool UpdateLockAtomic(Core::System& system, u32* out, KProcessAddress address, u32 if_zero, +bool UpdateLockAtomic(KernelCore& kernel, u32* out, KProcessAddress address, u32 if_zero, u32 new_orr_mask) { - auto& monitor = system.Monitor(); - const auto current_core = system.Kernel().CurrentPhysicalCoreIndex(); + auto& monitor = GetCurrentProcess(kernel).GetExclusiveMonitor(); + const auto current_core = kernel.CurrentPhysicalCoreIndex(); u32 expected{}; @@ -208,7 +208,7 @@ void KConditionVariable::SignalImpl(KThread* thread) { // TODO(bunnei): We should call CanAccessAtomic(..) here. can_access = true; if (can_access) [[likely]] { - UpdateLockAtomic(m_system, std::addressof(prev_tag), address, own_tag, + UpdateLockAtomic(m_kernel, std::addressof(prev_tag), address, own_tag, Svc::HandleWaitMask); } } diff --git a/src/core/hle/kernel/k_handle_table.h b/src/core/hle/kernel/k_handle_table.h index d7660630c..1bf68e6b0 100644 --- a/src/core/hle/kernel/k_handle_table.h +++ b/src/core/hle/kernel/k_handle_table.h @@ -90,8 +90,7 @@ public: // Handle pseudo-handles. if constexpr (std::derived_from<KProcess, T>) { if (handle == Svc::PseudoHandle::CurrentProcess) { - //! FIXME: this is the wrong process! - auto* const cur_process = m_kernel.ApplicationProcess(); + auto* const cur_process = GetCurrentProcessPointer(m_kernel); ASSERT(cur_process != nullptr); return cur_process; } diff --git a/src/core/hle/kernel/k_page_table_base.cpp b/src/core/hle/kernel/k_page_table_base.cpp index 423289145..8c1549559 100644 --- a/src/core/hle/kernel/k_page_table_base.cpp +++ b/src/core/hle/kernel/k_page_table_base.cpp @@ -434,7 +434,7 @@ Result KPageTableBase::InitializeForProcess(Svc::CreateProcessFlag as_type, bool void KPageTableBase::Finalize() { auto HostUnmapCallback = [&](KProcessAddress addr, u64 size) { if (Settings::IsFastmemEnabled()) { - m_system.DeviceMemory().buffer.Unmap(GetInteger(addr), size); + m_system.DeviceMemory().buffer.Unmap(GetInteger(addr), size, false); } }; @@ -5243,7 +5243,7 @@ Result KPageTableBase::MapPhysicalMemory(KProcessAddress address, size_t size) { // Unmap. R_ASSERT(this->Operate(updater.GetPageList(), cur_address, cur_pages, 0, false, unmap_properties, - OperationType::Unmap, true)); + OperationType::UnmapPhysical, true)); } // Check if we're done. @@ -5326,7 +5326,7 @@ Result KPageTableBase::MapPhysicalMemory(KProcessAddress address, size_t size) { // Map the papges. R_TRY(this->Operate(updater.GetPageList(), cur_address, map_pages, cur_pg, map_properties, - OperationType::MapFirstGroup, false)); + OperationType::MapFirstGroupPhysical, false)); } } @@ -5480,7 +5480,7 @@ Result KPageTableBase::UnmapPhysicalMemory(KProcessAddress address, size_t size) // Unmap. R_ASSERT(this->Operate(updater.GetPageList(), cur_address, cur_pages, 0, false, - unmap_properties, OperationType::Unmap, false)); + unmap_properties, OperationType::UnmapPhysical, false)); } // Check if we're done. @@ -5655,7 +5655,10 @@ Result KPageTableBase::Operate(PageLinkedList* page_list, KProcessAddress virt_a // or free them to the page list, and so it goes unused (along with page properties). switch (operation) { - case OperationType::Unmap: { + case OperationType::Unmap: + case OperationType::UnmapPhysical: { + const bool separate_heap = operation == OperationType::UnmapPhysical; + // Ensure that any pages we track are closed on exit. KPageGroup pages_to_close(m_kernel, this->GetBlockInfoManager()); SCOPE_EXIT({ pages_to_close.CloseAndReset(); }); @@ -5664,7 +5667,7 @@ Result KPageTableBase::Operate(PageLinkedList* page_list, KProcessAddress virt_a this->MakePageGroup(pages_to_close, virt_addr, num_pages); // Unmap. - m_memory->UnmapRegion(*m_impl, virt_addr, num_pages * PageSize); + m_memory->UnmapRegion(*m_impl, virt_addr, num_pages * PageSize, separate_heap); R_SUCCEED(); } @@ -5672,7 +5675,7 @@ Result KPageTableBase::Operate(PageLinkedList* page_list, KProcessAddress virt_a ASSERT(virt_addr != 0); ASSERT(Common::IsAligned(GetInteger(virt_addr), PageSize)); m_memory->MapMemoryRegion(*m_impl, virt_addr, num_pages * PageSize, phys_addr, - ConvertToMemoryPermission(properties.perm)); + ConvertToMemoryPermission(properties.perm), false); // Open references to pages, if we should. if (this->IsHeapPhysicalAddress(phys_addr)) { @@ -5711,16 +5714,19 @@ Result KPageTableBase::Operate(PageLinkedList* page_list, KProcessAddress virt_a switch (operation) { case OperationType::MapGroup: - case OperationType::MapFirstGroup: { + case OperationType::MapFirstGroup: + case OperationType::MapFirstGroupPhysical: { + const bool separate_heap = operation == OperationType::MapFirstGroupPhysical; + // We want to maintain a new reference to every page in the group. - KScopedPageGroup spg(page_group, operation != OperationType::MapFirstGroup); + KScopedPageGroup spg(page_group, operation == OperationType::MapGroup); for (const auto& node : page_group) { const size_t size{node.GetNumPages() * PageSize}; // Map the pages. m_memory->MapMemoryRegion(*m_impl, virt_addr, size, node.GetAddress(), - ConvertToMemoryPermission(properties.perm)); + ConvertToMemoryPermission(properties.perm), separate_heap); virt_addr += size; } diff --git a/src/core/hle/kernel/k_page_table_base.h b/src/core/hle/kernel/k_page_table_base.h index 556d230b3..077cafc96 100644 --- a/src/core/hle/kernel/k_page_table_base.h +++ b/src/core/hle/kernel/k_page_table_base.h @@ -104,6 +104,9 @@ protected: ChangePermissionsAndRefresh = 5, ChangePermissionsAndRefreshAndFlush = 6, Separate = 7, + + MapFirstGroupPhysical = 65000, + UnmapPhysical = 65001, }; static constexpr size_t MaxPhysicalMapAlignment = 1_GiB; diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp index 3a2635e1f..068e71dff 100644 --- a/src/core/hle/kernel/k_process.cpp +++ b/src/core/hle/kernel/k_process.cpp @@ -306,12 +306,16 @@ Result KProcess::Initialize(const Svc::CreateProcessParameter& params, const KPa False(params.flags & Svc::CreateProcessFlag::DisableDeviceAddressSpaceMerge); R_TRY(m_page_table.Initialize(as_type, enable_aslr, enable_das_merge, !enable_aslr, pool, params.code_address, params.code_num_pages * PageSize, - m_system_resource, res_limit, this->GetMemory(), 0)); + m_system_resource, res_limit, m_memory, 0)); } ON_RESULT_FAILURE_2 { m_page_table.Finalize(); }; + // Ensure our memory is initialized. + m_memory.SetCurrentPageTable(*this); + m_memory.SetGPUDirtyManagers(m_dirty_memory_managers); + // Ensure we can insert the code region. R_UNLESS(m_page_table.CanContain(params.code_address, params.code_num_pages * PageSize, KMemoryState::Code), @@ -399,12 +403,16 @@ Result KProcess::Initialize(const Svc::CreateProcessParameter& params, False(params.flags & Svc::CreateProcessFlag::DisableDeviceAddressSpaceMerge); R_TRY(m_page_table.Initialize(as_type, enable_aslr, enable_das_merge, !enable_aslr, pool, params.code_address, code_size, m_system_resource, res_limit, - this->GetMemory(), aslr_space_start)); + m_memory, aslr_space_start)); } ON_RESULT_FAILURE_2 { m_page_table.Finalize(); }; + // Ensure our memory is initialized. + m_memory.SetCurrentPageTable(*this); + m_memory.SetGPUDirtyManagers(m_dirty_memory_managers); + // Ensure we can insert the code region. R_UNLESS(m_page_table.CanContain(params.code_address, code_size, KMemoryState::Code), ResultInvalidMemoryRegion); @@ -1094,8 +1102,7 @@ void KProcess::UnpinThread(KThread* thread) { Result KProcess::GetThreadList(s32* out_num_threads, KProcessAddress out_thread_ids, s32 max_out_count) { - // TODO: use current memory reference - auto& memory = m_kernel.System().ApplicationMemory(); + auto& memory = this->GetMemory(); // Lock the list. KScopedLightLock lk(m_list_lock); @@ -1128,14 +1135,15 @@ void KProcess::Switch(KProcess* cur_process, KProcess* next_process) {} KProcess::KProcess(KernelCore& kernel) : KAutoObjectWithSlabHeapAndContainer(kernel), m_page_table{kernel}, m_state_lock{kernel}, m_list_lock{kernel}, m_cond_var{kernel.System()}, m_address_arbiter{kernel.System()}, - m_handle_table{kernel} {} + m_handle_table{kernel}, m_dirty_memory_managers{}, + m_exclusive_monitor{}, m_memory{kernel.System()} {} KProcess::~KProcess() = default; Result KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std::size_t code_size, KProcessAddress aslr_space_start, bool is_hbl) { // Create a resource limit for the process. - const auto physical_memory_size = - m_kernel.MemoryManager().GetSize(Kernel::KMemoryManager::Pool::Application); + const auto pool = static_cast<KMemoryManager::Pool>(metadata.GetPoolPartition()); + const auto physical_memory_size = m_kernel.MemoryManager().GetSize(pool); auto* res_limit = Kernel::CreateResourceLimitForProcess(m_kernel.System(), physical_memory_size); @@ -1146,8 +1154,10 @@ Result KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std: Svc::CreateProcessFlag flag{}; u64 code_address{}; - // We are an application. - flag |= Svc::CreateProcessFlag::IsApplication; + // Determine if we are an application. + if (pool == KMemoryManager::Pool::Application) { + flag |= Svc::CreateProcessFlag::IsApplication; + } // If we are 64-bit, create as such. if (metadata.Is64BitProgram()) { @@ -1196,8 +1206,8 @@ Result KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std: std::memcpy(params.name.data(), name.data(), sizeof(params.name)); // Initialize for application process. - R_TRY(this->Initialize(params, metadata.GetKernelCapabilities(), res_limit, - KMemoryManager::Pool::Application, aslr_space_start)); + R_TRY(this->Initialize(params, metadata.GetKernelCapabilities(), res_limit, pool, + aslr_space_start)); // Assign remaining properties. m_is_hbl = is_hbl; @@ -1223,22 +1233,25 @@ void KProcess::LoadModule(CodeSet code_set, KProcessAddress base_addr) { ReprotectSegment(code_set.DataSegment(), Svc::MemoryPermission::ReadWrite); #ifdef HAS_NCE - if (Settings::IsNceEnabled()) { + if (this->IsApplication() && Settings::IsNceEnabled()) { auto& buffer = m_kernel.System().DeviceMemory().buffer; const auto& code = code_set.CodeSegment(); const auto& patch = code_set.PatchSegment(); - buffer.Protect(GetInteger(base_addr + code.addr), code.size, true, true, true); - buffer.Protect(GetInteger(base_addr + patch.addr), patch.size, true, true, true); + buffer.Protect(GetInteger(base_addr + code.addr), code.size, + Common::MemoryPermission::Read | Common::MemoryPermission::Execute); + buffer.Protect(GetInteger(base_addr + patch.addr), patch.size, + Common::MemoryPermission::Read | Common::MemoryPermission::Execute); ReprotectSegment(code_set.PatchSegment(), Svc::MemoryPermission::None); } #endif } void KProcess::InitializeInterfaces() { - this->GetMemory().SetCurrentPageTable(*this); + m_exclusive_monitor = + Core::MakeExclusiveMonitor(this->GetMemory(), Core::Hardware::NUM_CPU_CORES); #ifdef HAS_NCE - if (this->Is64Bit() && Settings::IsNceEnabled()) { + if (this->IsApplication() && Settings::IsNceEnabled()) { for (size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { m_arm_interfaces[i] = std::make_unique<Core::ArmNce>(m_kernel.System(), true, i); } @@ -1248,13 +1261,13 @@ void KProcess::InitializeInterfaces() { for (size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { m_arm_interfaces[i] = std::make_unique<Core::ArmDynarmic64>( m_kernel.System(), m_kernel.IsMulticore(), this, - static_cast<Core::DynarmicExclusiveMonitor&>(m_kernel.GetExclusiveMonitor()), i); + static_cast<Core::DynarmicExclusiveMonitor&>(*m_exclusive_monitor), i); } } else { for (size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { m_arm_interfaces[i] = std::make_unique<Core::ArmDynarmic32>( m_kernel.System(), m_kernel.IsMulticore(), this, - static_cast<Core::DynarmicExclusiveMonitor&>(m_kernel.GetExclusiveMonitor()), i); + static_cast<Core::DynarmicExclusiveMonitor&>(*m_exclusive_monitor), i); } } } @@ -1305,9 +1318,10 @@ bool KProcess::RemoveWatchpoint(KProcessAddress addr, u64 size, DebugWatchpointT return true; } -Core::Memory::Memory& KProcess::GetMemory() const { - // TODO: per-process memory - return m_kernel.System().ApplicationMemory(); +void KProcess::GatherGPUDirtyMemory(std::function<void(VAddr, size_t)>& callback) { + for (auto& manager : m_dirty_memory_managers) { + manager.Gather(callback); + } } } // namespace Kernel diff --git a/src/core/hle/kernel/k_process.h b/src/core/hle/kernel/k_process.h index 4b114e39b..53c0e3316 100644 --- a/src/core/hle/kernel/k_process.h +++ b/src/core/hle/kernel/k_process.h @@ -7,6 +7,7 @@ #include "core/arm/arm_interface.h" #include "core/file_sys/program_metadata.h" +#include "core/gpu_dirty_memory_manager.h" #include "core/hle/kernel/code_set.h" #include "core/hle/kernel/k_address_arbiter.h" #include "core/hle/kernel/k_capabilities.h" @@ -17,6 +18,7 @@ #include "core/hle/kernel/k_system_resource.h" #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/k_thread_local_page.h" +#include "core/memory.h" namespace Kernel { @@ -126,6 +128,9 @@ private: #ifdef HAS_NCE std::unordered_map<u64, u64> m_post_handlers{}; #endif + std::array<Core::GPUDirtyMemoryManager, Core::Hardware::NUM_CPU_CORES> m_dirty_memory_managers; + std::unique_ptr<Core::ExclusiveMonitor> m_exclusive_monitor; + Core::Memory::Memory m_memory; private: Result StartTermination(); @@ -502,7 +507,15 @@ public: void InitializeInterfaces(); - Core::Memory::Memory& GetMemory() const; + Core::Memory::Memory& GetMemory() { + return m_memory; + } + + void GatherGPUDirtyMemory(std::function<void(VAddr, size_t)>& callback); + + Core::ExclusiveMonitor& GetExclusiveMonitor() const { + return *m_exclusive_monitor; + } public: // Overridden parent functions. diff --git a/src/core/hle/kernel/k_server_session.cpp b/src/core/hle/kernel/k_server_session.cpp index e33a88e24..adaabdd6d 100644 --- a/src/core/hle/kernel/k_server_session.cpp +++ b/src/core/hle/kernel/k_server_session.cpp @@ -8,6 +8,7 @@ #include "common/common_types.h" #include "common/logging/log.h" #include "common/scope_exit.h" +#include "common/scratch_buffer.h" #include "core/core.h" #include "core/core_timing.h" #include "core/hle/kernel/k_client_port.h" @@ -29,12 +30,138 @@ namespace Kernel { namespace { +constexpr inline size_t PointerTransferBufferAlignment = 0x10; +constexpr inline size_t ReceiveListDataSize = + MessageBuffer::MessageHeader::ReceiveListCountType_CountMax * + MessageBuffer::ReceiveListEntry::GetDataSize() / sizeof(u32); + +using ThreadQueueImplForKServerSessionRequest = KThreadQueue; + +class ReceiveList { +public: + static constexpr int GetEntryCount(const MessageBuffer::MessageHeader& header) { + const auto count = header.GetReceiveListCount(); + switch (count) { + case MessageBuffer::MessageHeader::ReceiveListCountType_None: + return 0; + case MessageBuffer::MessageHeader::ReceiveListCountType_ToMessageBuffer: + return 0; + case MessageBuffer::MessageHeader::ReceiveListCountType_ToSingleBuffer: + return 1; + default: + return count - MessageBuffer::MessageHeader::ReceiveListCountType_CountOffset; + } + } + + explicit ReceiveList(const u32* dst_msg, uint64_t dst_address, + KProcessPageTable& dst_page_table, + const MessageBuffer::MessageHeader& dst_header, + const MessageBuffer::SpecialHeader& dst_special_header, size_t msg_size, + size_t out_offset, s32 dst_recv_list_idx, bool is_tls) { + m_recv_list_count = dst_header.GetReceiveListCount(); + m_msg_buffer_end = dst_address + sizeof(u32) * out_offset; + m_msg_buffer_space_end = dst_address + msg_size; + + // NOTE: Nintendo calculates the receive list index here using the special header. + // We pre-calculate it in the caller, and pass it as a parameter. + (void)dst_special_header; + + const u32* recv_list = dst_msg + dst_recv_list_idx; + const auto entry_count = GetEntryCount(dst_header); + + if (is_tls) { + // Messages from TLS to TLS are contained within one page. + std::memcpy(m_data.data(), recv_list, + entry_count * MessageBuffer::ReceiveListEntry::GetDataSize()); + } else { + // If any buffer is not from TLS, perform a normal read instead. + uint64_t cur_addr = dst_address + dst_recv_list_idx * sizeof(u32); + dst_page_table.GetMemory().ReadBlock( + cur_addr, m_data.data(), + entry_count * MessageBuffer::ReceiveListEntry::GetDataSize()); + } + } + + bool IsIndex() const { + return m_recv_list_count > + static_cast<s32>(MessageBuffer::MessageHeader::ReceiveListCountType_CountOffset); + } + + bool IsToMessageBuffer() const { + return m_recv_list_count == + MessageBuffer::MessageHeader::ReceiveListCountType_ToMessageBuffer; + } + + void GetBuffer(uint64_t& out, size_t size, int& key) const { + switch (m_recv_list_count) { + case MessageBuffer::MessageHeader::ReceiveListCountType_None: { + out = 0; + break; + } + case MessageBuffer::MessageHeader::ReceiveListCountType_ToMessageBuffer: { + const uint64_t buf = + Common::AlignUp(m_msg_buffer_end + key, PointerTransferBufferAlignment); + + if ((buf < buf + size) && (buf + size <= m_msg_buffer_space_end)) { + out = buf; + key = static_cast<int>(buf + size - m_msg_buffer_end); + } else { + out = 0; + } + break; + } + case MessageBuffer::MessageHeader::ReceiveListCountType_ToSingleBuffer: { + const MessageBuffer::ReceiveListEntry entry(m_data[0], m_data[1]); + const uint64_t buf = + Common::AlignUp(entry.GetAddress() + key, PointerTransferBufferAlignment); + + const uint64_t entry_addr = entry.GetAddress(); + const size_t entry_size = entry.GetSize(); + + if ((buf < buf + size) && (entry_addr < entry_addr + entry_size) && + (buf + size <= entry_addr + entry_size)) { + out = buf; + key = static_cast<int>(buf + size - entry_addr); + } else { + out = 0; + } + break; + } + default: { + if (key < m_recv_list_count - + static_cast<s32>( + MessageBuffer::MessageHeader::ReceiveListCountType_CountOffset)) { + const MessageBuffer::ReceiveListEntry entry(m_data[2 * key + 0], + m_data[2 * key + 1]); + + const uintptr_t entry_addr = entry.GetAddress(); + const size_t entry_size = entry.GetSize(); + + if ((entry_addr < entry_addr + entry_size) && (entry_size >= size)) { + out = entry_addr; + } + } else { + out = 0; + } + break; + } + } + } + +private: + std::array<u32, ReceiveListDataSize> m_data; + s32 m_recv_list_count; + uint64_t m_msg_buffer_end; + uint64_t m_msg_buffer_space_end; +}; + template <bool MoveHandleAllowed> -Result ProcessMessageSpecialData(KProcess& dst_process, KProcess& src_process, KThread& src_thread, - MessageBuffer& dst_msg, const MessageBuffer& src_msg, - MessageBuffer::SpecialHeader& src_special_header) { +Result ProcessMessageSpecialData(s32& offset, KProcess& dst_process, KProcess& src_process, + KThread& src_thread, const MessageBuffer& dst_msg, + const MessageBuffer& src_msg, + const MessageBuffer::SpecialHeader& src_special_header) { // Copy the special header to the destination. - s32 offset = dst_msg.Set(src_special_header); + offset = dst_msg.Set(src_special_header); // Copy the process ID. if (src_special_header.GetHasProcessId()) { @@ -110,6 +237,102 @@ Result ProcessMessageSpecialData(KProcess& dst_process, KProcess& src_process, K R_RETURN(result); } +Result ProcessReceiveMessagePointerDescriptors(int& offset, int& pointer_key, + KProcessPageTable& dst_page_table, + KProcessPageTable& src_page_table, + const MessageBuffer& dst_msg, + const MessageBuffer& src_msg, + const ReceiveList& dst_recv_list, bool dst_user) { + // Get the offset at the start of processing. + const int cur_offset = offset; + + // Get the pointer desc. + MessageBuffer::PointerDescriptor src_desc(src_msg, cur_offset); + offset += static_cast<int>(MessageBuffer::PointerDescriptor::GetDataSize() / sizeof(u32)); + + // Extract address/size. + const uint64_t src_pointer = src_desc.GetAddress(); + const size_t recv_size = src_desc.GetSize(); + uint64_t recv_pointer = 0; + + // Process the buffer, if it has a size. + if (recv_size > 0) { + // If using indexing, set index. + if (dst_recv_list.IsIndex()) { + pointer_key = src_desc.GetIndex(); + } + + // Get the buffer. + dst_recv_list.GetBuffer(recv_pointer, recv_size, pointer_key); + R_UNLESS(recv_pointer != 0, ResultOutOfResource); + + // Perform the pointer data copy. + if (dst_user) { + R_TRY(src_page_table.CopyMemoryFromHeapToHeapWithoutCheckDestination( + dst_page_table, recv_pointer, recv_size, KMemoryState::FlagReferenceCounted, + KMemoryState::FlagReferenceCounted, + KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite, + KMemoryAttribute::Uncached | KMemoryAttribute::Locked, KMemoryAttribute::Locked, + src_pointer, KMemoryState::FlagLinearMapped, KMemoryState::FlagLinearMapped, + KMemoryPermission::UserRead, KMemoryAttribute::Uncached, KMemoryAttribute::None)); + } else { + R_TRY(src_page_table.CopyMemoryFromLinearToUser( + recv_pointer, recv_size, src_pointer, KMemoryState::FlagLinearMapped, + KMemoryState::FlagLinearMapped, KMemoryPermission::UserRead, + KMemoryAttribute::Uncached, KMemoryAttribute::None)); + } + } + + // Set the output descriptor. + dst_msg.Set(cur_offset, MessageBuffer::PointerDescriptor(reinterpret_cast<void*>(recv_pointer), + recv_size, src_desc.GetIndex())); + + R_SUCCEED(); +} + +constexpr Result GetMapAliasMemoryState(KMemoryState& out, + MessageBuffer::MapAliasDescriptor::Attribute attr) { + switch (attr) { + case MessageBuffer::MapAliasDescriptor::Attribute::Ipc: + out = KMemoryState::Ipc; + break; + case MessageBuffer::MapAliasDescriptor::Attribute::NonSecureIpc: + out = KMemoryState::NonSecureIpc; + break; + case MessageBuffer::MapAliasDescriptor::Attribute::NonDeviceIpc: + out = KMemoryState::NonDeviceIpc; + break; + default: + R_THROW(ResultInvalidCombination); + } + + R_SUCCEED(); +} + +constexpr Result GetMapAliasTestStateAndAttributeMask(KMemoryState& out_state, + KMemoryAttribute& out_attr_mask, + KMemoryState state) { + switch (state) { + case KMemoryState::Ipc: + out_state = KMemoryState::FlagCanUseIpc; + out_attr_mask = + KMemoryAttribute::Uncached | KMemoryAttribute::DeviceShared | KMemoryAttribute::Locked; + break; + case KMemoryState::NonSecureIpc: + out_state = KMemoryState::FlagCanUseNonSecureIpc; + out_attr_mask = KMemoryAttribute::Uncached | KMemoryAttribute::Locked; + break; + case KMemoryState::NonDeviceIpc: + out_state = KMemoryState::FlagCanUseNonDeviceIpc; + out_attr_mask = KMemoryAttribute::Uncached | KMemoryAttribute::Locked; + break; + default: + R_THROW(ResultInvalidCombination); + } + + R_SUCCEED(); +} + void CleanupSpecialData(KProcess& dst_process, u32* dst_msg_ptr, size_t dst_buffer_size) { // Parse the message. const MessageBuffer dst_msg(dst_msg_ptr, dst_buffer_size); @@ -144,166 +367,855 @@ void CleanupSpecialData(KProcess& dst_process, u32* dst_msg_ptr, size_t dst_buff } } -} // namespace +Result CleanupServerHandles(KernelCore& kernel, uint64_t message, size_t buffer_size, + KPhysicalAddress message_paddr) { + // Server is assumed to be current thread. + KThread& thread = GetCurrentThread(kernel); -using ThreadQueueImplForKServerSessionRequest = KThreadQueue; + // Get the linear message pointer. + u32* msg_ptr; + if (message) { + msg_ptr = kernel.System().DeviceMemory().GetPointer<u32>(message_paddr); + } else { + msg_ptr = GetCurrentMemory(kernel).GetPointer<u32>(thread.GetTlsAddress()); + buffer_size = MessageBufferSize; + message = GetInteger(thread.GetTlsAddress()); + } -KServerSession::KServerSession(KernelCore& kernel) - : KSynchronizationObject{kernel}, m_lock{m_kernel} {} + // Parse the message. + const MessageBuffer msg(msg_ptr, buffer_size); + const MessageBuffer::MessageHeader header(msg); + const MessageBuffer::SpecialHeader special_header(msg, header); -KServerSession::~KServerSession() = default; + // Check that the size is big enough. + R_UNLESS(MessageBuffer::GetMessageBufferSize(header, special_header) <= buffer_size, + ResultInvalidCombination); + + // If there's a special header, there may be move handles we need to close. + if (header.GetHasSpecialHeader()) { + // Determine the offset to the start of handles. + auto offset = msg.GetSpecialDataIndex(header, special_header); + if (special_header.GetHasProcessId()) { + offset += static_cast<int>(sizeof(u64) / sizeof(u32)); + } + if (auto copy_count = special_header.GetCopyHandleCount(); copy_count > 0) { + offset += static_cast<int>((sizeof(Svc::Handle) * copy_count) / sizeof(u32)); + } -void KServerSession::Destroy() { - m_parent->OnServerClosed(); + // Get the handle table. + auto& handle_table = thread.GetOwnerProcess()->GetHandleTable(); - this->CleanupRequests(); + // Close the handles. + for (auto i = 0; i < special_header.GetMoveHandleCount(); ++i) { + handle_table.Remove(msg.GetHandle(offset)); + offset += static_cast<int>(sizeof(Svc::Handle) / sizeof(u32)); + } + } - m_parent->Close(); + R_SUCCEED(); } -void KServerSession::OnClientClosed() { - KScopedLightLock lk{m_lock}; +Result CleanupServerMap(KSessionRequest* request, KProcess* server_process) { + // If there's no server process, there's nothing to clean up. + R_SUCCEED_IF(server_process == nullptr); - // Handle any pending requests. - KSessionRequest* prev_request = nullptr; - while (true) { - // Declare variables for processing the request. - KSessionRequest* request = nullptr; - KEvent* event = nullptr; - KThread* thread = nullptr; - bool cur_request = false; - bool terminate = false; + // Get the page table. + auto& server_page_table = server_process->GetPageTable(); - // Get the next request. - { - KScopedSchedulerLock sl{m_kernel}; + // Cleanup Send mappings. + for (size_t i = 0; i < request->GetSendCount(); ++i) { + R_TRY(server_page_table.CleanupForIpcServer(request->GetSendServerAddress(i), + request->GetSendSize(i), + request->GetSendMemoryState(i))); + } - if (m_current_request != nullptr && m_current_request != prev_request) { - // Set the request, open a reference as we process it. - request = m_current_request; - request->Open(); - cur_request = true; + // Cleanup Receive mappings. + for (size_t i = 0; i < request->GetReceiveCount(); ++i) { + R_TRY(server_page_table.CleanupForIpcServer(request->GetReceiveServerAddress(i), + request->GetReceiveSize(i), + request->GetReceiveMemoryState(i))); + } - // Get thread and event for the request. - thread = request->GetThread(); - event = request->GetEvent(); + // Cleanup Exchange mappings. + for (size_t i = 0; i < request->GetExchangeCount(); ++i) { + R_TRY(server_page_table.CleanupForIpcServer(request->GetExchangeServerAddress(i), + request->GetExchangeSize(i), + request->GetExchangeMemoryState(i))); + } - // If the thread is terminating, handle that. - if (thread->IsTerminationRequested()) { - request->ClearThread(); - request->ClearEvent(); - terminate = true; - } + R_SUCCEED(); +} - prev_request = request; - } else if (!m_request_list.empty()) { - // Pop the request from the front of the list. - request = std::addressof(m_request_list.front()); - m_request_list.pop_front(); +Result CleanupClientMap(KSessionRequest* request, KProcessPageTable* client_page_table) { + // If there's no client page table, there's nothing to clean up. + R_SUCCEED_IF(client_page_table == nullptr); - // Get thread and event for the request. - thread = request->GetThread(); - event = request->GetEvent(); - } + // Cleanup Send mappings. + for (size_t i = 0; i < request->GetSendCount(); ++i) { + R_TRY(client_page_table->CleanupForIpcClient(request->GetSendClientAddress(i), + request->GetSendSize(i), + request->GetSendMemoryState(i))); + } + + // Cleanup Receive mappings. + for (size_t i = 0; i < request->GetReceiveCount(); ++i) { + R_TRY(client_page_table->CleanupForIpcClient(request->GetReceiveClientAddress(i), + request->GetReceiveSize(i), + request->GetReceiveMemoryState(i))); + } + + // Cleanup Exchange mappings. + for (size_t i = 0; i < request->GetExchangeCount(); ++i) { + R_TRY(client_page_table->CleanupForIpcClient(request->GetExchangeClientAddress(i), + request->GetExchangeSize(i), + request->GetExchangeMemoryState(i))); + } + + R_SUCCEED(); +} + +Result CleanupMap(KSessionRequest* request, KProcess* server_process, + KProcessPageTable* client_page_table) { + // Cleanup the server map. + R_TRY(CleanupServerMap(request, server_process)); + + // Cleanup the client map. + R_TRY(CleanupClientMap(request, client_page_table)); + + R_SUCCEED(); +} + +Result ProcessReceiveMessageMapAliasDescriptors(int& offset, KProcessPageTable& dst_page_table, + KProcessPageTable& src_page_table, + const MessageBuffer& dst_msg, + const MessageBuffer& src_msg, + KSessionRequest* request, KMemoryPermission perm, + bool send) { + // Get the offset at the start of processing. + const int cur_offset = offset; + + // Get the map alias descriptor. + MessageBuffer::MapAliasDescriptor src_desc(src_msg, cur_offset); + offset += static_cast<int>(MessageBuffer::MapAliasDescriptor::GetDataSize() / sizeof(u32)); + + // Extract address/size. + const KProcessAddress src_address = src_desc.GetAddress(); + const size_t size = src_desc.GetSize(); + KProcessAddress dst_address = 0; + + // Determine the result memory state. + KMemoryState dst_state; + R_TRY(GetMapAliasMemoryState(dst_state, src_desc.GetAttribute())); + + // Process the buffer, if it has a size. + if (size > 0) { + // Set up the source pages for ipc. + R_TRY(dst_page_table.SetupForIpc(std::addressof(dst_address), size, src_address, + src_page_table, perm, dst_state, send)); + + // Ensure that we clean up on failure. + ON_RESULT_FAILURE { + dst_page_table.CleanupForIpcServer(dst_address, size, dst_state); + src_page_table.CleanupForIpcClient(src_address, size, dst_state); + }; + + // Push the appropriate mapping. + if (perm == KMemoryPermission::UserRead) { + R_TRY(request->PushSend(src_address, dst_address, size, dst_state)); + } else if (send) { + R_TRY(request->PushExchange(src_address, dst_address, size, dst_state)); + } else { + R_TRY(request->PushReceive(src_address, dst_address, size, dst_state)); } + } - // If there are no requests, we're done. - if (request == nullptr) { - break; + // Set the output descriptor. + dst_msg.Set(cur_offset, + MessageBuffer::MapAliasDescriptor(reinterpret_cast<void*>(GetInteger(dst_address)), + size, src_desc.GetAttribute())); + + R_SUCCEED(); +} + +Result ReceiveMessage(KernelCore& kernel, bool& recv_list_broken, uint64_t dst_message_buffer, + size_t dst_buffer_size, KPhysicalAddress dst_message_paddr, + KThread& src_thread, uint64_t src_message_buffer, size_t src_buffer_size, + KServerSession* session, KSessionRequest* request) { + // Prepare variables for receive. + KThread& dst_thread = GetCurrentThread(kernel); + KProcess& dst_process = *(dst_thread.GetOwnerProcess()); + KProcess& src_process = *(src_thread.GetOwnerProcess()); + auto& dst_page_table = dst_process.GetPageTable(); + auto& src_page_table = src_process.GetPageTable(); + + // NOTE: Session is used only for debugging, and so may go unused. + (void)session; + + // The receive list is initially not broken. + recv_list_broken = false; + + // Set the server process for the request. + request->SetServerProcess(std::addressof(dst_process)); + + // Determine the message buffers. + u32 *dst_msg_ptr, *src_msg_ptr; + bool dst_user, src_user; + + if (dst_message_buffer) { + dst_msg_ptr = kernel.System().DeviceMemory().GetPointer<u32>(dst_message_paddr); + dst_user = true; + } else { + dst_msg_ptr = dst_page_table.GetMemory().GetPointer<u32>(dst_thread.GetTlsAddress()); + dst_buffer_size = MessageBufferSize; + dst_message_buffer = GetInteger(dst_thread.GetTlsAddress()); + dst_user = false; + } + + if (src_message_buffer) { + // NOTE: Nintendo does not check the result of this GetPhysicalAddress call. + src_msg_ptr = src_page_table.GetMemory().GetPointer<u32>(src_message_buffer); + src_user = true; + } else { + src_msg_ptr = src_page_table.GetMemory().GetPointer<u32>(src_thread.GetTlsAddress()); + src_buffer_size = MessageBufferSize; + src_message_buffer = GetInteger(src_thread.GetTlsAddress()); + src_user = false; + } + + // Parse the headers. + const MessageBuffer dst_msg(dst_msg_ptr, dst_buffer_size); + const MessageBuffer src_msg(src_msg_ptr, src_buffer_size); + const MessageBuffer::MessageHeader dst_header(dst_msg); + const MessageBuffer::MessageHeader src_header(src_msg); + const MessageBuffer::SpecialHeader dst_special_header(dst_msg, dst_header); + const MessageBuffer::SpecialHeader src_special_header(src_msg, src_header); + + // Get the end of the source message. + const size_t src_end_offset = + MessageBuffer::GetRawDataIndex(src_header, src_special_header) + src_header.GetRawCount(); + + // Ensure that the headers fit. + R_UNLESS(MessageBuffer::GetMessageBufferSize(dst_header, dst_special_header) <= dst_buffer_size, + ResultInvalidCombination); + R_UNLESS(MessageBuffer::GetMessageBufferSize(src_header, src_special_header) <= src_buffer_size, + ResultInvalidCombination); + + // Ensure the receive list offset is after the end of raw data. + if (dst_header.GetReceiveListOffset()) { + R_UNLESS(dst_header.GetReceiveListOffset() >= + MessageBuffer::GetRawDataIndex(dst_header, dst_special_header) + + dst_header.GetRawCount(), + ResultInvalidCombination); + } + + // Ensure that the destination buffer is big enough to receive the source. + R_UNLESS(dst_buffer_size >= src_end_offset * sizeof(u32), ResultMessageTooLarge); + + // Get the receive list. + const s32 dst_recv_list_idx = + MessageBuffer::GetReceiveListIndex(dst_header, dst_special_header); + ReceiveList dst_recv_list(dst_msg_ptr, dst_message_buffer, dst_page_table, dst_header, + dst_special_header, dst_buffer_size, src_end_offset, + dst_recv_list_idx, !dst_user); + + // Ensure that the source special header isn't invalid. + const bool src_has_special_header = src_header.GetHasSpecialHeader(); + if (src_has_special_header) { + // Sending move handles from client -> server is not allowed. + R_UNLESS(src_special_header.GetMoveHandleCount() == 0, ResultInvalidCombination); + } + + // Prepare for further processing. + int pointer_key = 0; + int offset = dst_msg.Set(src_header); + + // Set up a guard to make sure that we end up in a clean state on error. + ON_RESULT_FAILURE { + // Cleanup mappings. + CleanupMap(request, std::addressof(dst_process), std::addressof(src_page_table)); + + // Cleanup special data. + if (src_header.GetHasSpecialHeader()) { + CleanupSpecialData(dst_process, dst_msg_ptr, dst_buffer_size); } - // All requests must have threads. - ASSERT(thread != nullptr); + // Cleanup the header if the receive list isn't broken. + if (!recv_list_broken) { + dst_msg.Set(dst_header); + if (dst_header.GetHasSpecialHeader()) { + dst_msg.Set(dst_special_header); + } + } + }; + + // Process any special data. + if (src_header.GetHasSpecialHeader()) { + // After we process, make sure we track whether the receive list is broken. + SCOPE_EXIT({ + if (offset > dst_recv_list_idx) { + recv_list_broken = true; + } + }); - // Ensure that we close the request when done. - SCOPE_EXIT({ request->Close(); }); + // Process special data. + R_TRY(ProcessMessageSpecialData<false>(offset, dst_process, src_process, src_thread, + dst_msg, src_msg, src_special_header)); + } - // If we're terminating, close a reference to the thread and event. - if (terminate) { - thread->Close(); - if (event != nullptr) { - event->Close(); + // Process any pointer buffers. + for (auto i = 0; i < src_header.GetPointerCount(); ++i) { + // After we process, make sure we track whether the receive list is broken. + SCOPE_EXIT({ + if (offset > dst_recv_list_idx) { + recv_list_broken = true; + } + }); + + R_TRY(ProcessReceiveMessagePointerDescriptors( + offset, pointer_key, dst_page_table, src_page_table, dst_msg, src_msg, dst_recv_list, + dst_user && dst_header.GetReceiveListCount() == + MessageBuffer::MessageHeader::ReceiveListCountType_ToMessageBuffer)); + } + + // Process any map alias buffers. + for (auto i = 0; i < src_header.GetMapAliasCount(); ++i) { + // After we process, make sure we track whether the receive list is broken. + SCOPE_EXIT({ + if (offset > dst_recv_list_idx) { + recv_list_broken = true; } + }); + + // We process in order send, recv, exch. Buffers after send (recv/exch) are ReadWrite. + const KMemoryPermission perm = (i >= src_header.GetSendCount()) + ? KMemoryPermission::UserReadWrite + : KMemoryPermission::UserRead; + + // Buffer is send if it is send or exch. + const bool send = (i < src_header.GetSendCount()) || + (i >= src_header.GetSendCount() + src_header.GetReceiveCount()); + + R_TRY(ProcessReceiveMessageMapAliasDescriptors(offset, dst_page_table, src_page_table, + dst_msg, src_msg, request, perm, send)); + } + + // Process any raw data. + if (const auto raw_count = src_header.GetRawCount(); raw_count != 0) { + // After we process, make sure we track whether the receive list is broken. + SCOPE_EXIT({ + if (offset + raw_count > dst_recv_list_idx) { + recv_list_broken = true; + } + }); + + // Get the offset and size. + const size_t offset_words = offset * sizeof(u32); + const size_t raw_size = raw_count * sizeof(u32); + + if (!dst_user && !src_user) { + // Fast case is TLS -> TLS, do raw memcpy if we can. + std::memcpy(dst_msg_ptr + offset, src_msg_ptr + offset, raw_size); + } else if (dst_user) { + // Determine how much fast size we can copy. + const size_t max_fast_size = std::min<size_t>(offset_words + raw_size, PageSize); + const size_t fast_size = max_fast_size - offset_words; + + // Determine source state; if user buffer, we require heap, and otherwise only linear + // mapped (to enable tls use). + const auto src_state = + src_user ? KMemoryState::FlagReferenceCounted : KMemoryState::FlagLinearMapped; + + // Determine the source permission. User buffer should be unmapped + read, TLS should be + // user readable. + const KMemoryPermission src_perm = static_cast<KMemoryPermission>( + src_user ? KMemoryPermission::NotMapped | KMemoryPermission::KernelRead + : KMemoryPermission::UserRead); + + // Perform the fast part of the copy. + R_TRY(src_page_table.CopyMemoryFromLinearToKernel( + dst_msg_ptr + offset, fast_size, src_message_buffer + offset_words, src_state, + src_state, src_perm, KMemoryAttribute::Uncached, KMemoryAttribute::None)); + + // If the fast part of the copy didn't get everything, perform the slow part of the + // copy. + if (fast_size < raw_size) { + R_TRY(src_page_table.CopyMemoryFromHeapToHeap( + dst_page_table, dst_message_buffer + max_fast_size, raw_size - fast_size, + KMemoryState::FlagReferenceCounted, KMemoryState::FlagReferenceCounted, + KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite, + KMemoryAttribute::Uncached | KMemoryAttribute::Locked, KMemoryAttribute::Locked, + src_message_buffer + max_fast_size, src_state, src_state, src_perm, + KMemoryAttribute::Uncached, KMemoryAttribute::None)); + } + } else /* if (src_user) */ { + // The source is a user buffer, so it should be unmapped + readable. + constexpr KMemoryPermission SourcePermission = static_cast<KMemoryPermission>( + KMemoryPermission::NotMapped | KMemoryPermission::KernelRead); + + // Copy the memory. + R_TRY(src_page_table.CopyMemoryFromLinearToUser( + dst_message_buffer + offset_words, raw_size, src_message_buffer + offset_words, + KMemoryState::FlagReferenceCounted, KMemoryState::FlagReferenceCounted, + SourcePermission, KMemoryAttribute::Uncached, KMemoryAttribute::None)); } + } - // If we need to, reply. - if (event != nullptr && !cur_request) { - // There must be no mappings. - ASSERT(request->GetSendCount() == 0); - ASSERT(request->GetReceiveCount() == 0); - ASSERT(request->GetExchangeCount() == 0); + // We succeeded! + R_SUCCEED(); +} - // // Get the process and page table. - // KProcess *client_process = thread->GetOwnerProcess(); - // auto& client_pt = client_process->GetPageTable(); +Result ProcessSendMessageReceiveMapping(KProcessPageTable& src_page_table, + KProcessPageTable& dst_page_table, + KProcessAddress client_address, + KProcessAddress server_address, size_t size, + KMemoryState src_state) { + // If the size is zero, there's nothing to process. + R_SUCCEED_IF(size == 0); + + // Get the memory state and attribute mask to test. + KMemoryState test_state; + KMemoryAttribute test_attr_mask; + R_TRY(GetMapAliasTestStateAndAttributeMask(test_state, test_attr_mask, src_state)); + + // Determine buffer extents. + KProcessAddress aligned_dst_start = Common::AlignDown(GetInteger(client_address), PageSize); + KProcessAddress aligned_dst_end = Common::AlignUp(GetInteger(client_address) + size, PageSize); + KProcessAddress mapping_dst_start = Common::AlignUp(GetInteger(client_address), PageSize); + KProcessAddress mapping_dst_end = + Common::AlignDown(GetInteger(client_address) + size, PageSize); + + KProcessAddress mapping_src_end = + Common::AlignDown(GetInteger(server_address) + size, PageSize); + + // If the start of the buffer is unaligned, handle that. + if (aligned_dst_start != mapping_dst_start) { + ASSERT(client_address < mapping_dst_start); + const size_t copy_size = std::min<size_t>(size, mapping_dst_start - client_address); + R_TRY(dst_page_table.CopyMemoryFromUserToLinear( + client_address, copy_size, test_state, test_state, KMemoryPermission::UserReadWrite, + test_attr_mask, KMemoryAttribute::None, server_address)); + } - // // Reply to the request. - // ReplyAsyncError(client_process, request->GetAddress(), request->GetSize(), - // ResultSessionClosed); + // If the end of the buffer is unaligned, handle that. + if (mapping_dst_end < aligned_dst_end && + (aligned_dst_start == mapping_dst_start || aligned_dst_start < mapping_dst_end)) { + const size_t copy_size = client_address + size - mapping_dst_end; + R_TRY(dst_page_table.CopyMemoryFromUserToLinear( + mapping_dst_end, copy_size, test_state, test_state, KMemoryPermission::UserReadWrite, + test_attr_mask, KMemoryAttribute::None, mapping_src_end)); + } - // // Unlock the buffer. - // // NOTE: Nintendo does not check the result of this. - // client_pt.UnlockForIpcUserBuffer(request->GetAddress(), request->GetSize()); + R_SUCCEED(); +} - // Signal the event. - event->Signal(); +Result ProcessSendMessagePointerDescriptors(int& offset, int& pointer_key, + KProcessPageTable& src_page_table, + KProcessPageTable& dst_page_table, + const MessageBuffer& dst_msg, + const MessageBuffer& src_msg, + const ReceiveList& dst_recv_list, bool dst_user) { + // Get the offset at the start of processing. + const int cur_offset = offset; + + // Get the pointer desc. + MessageBuffer::PointerDescriptor src_desc(src_msg, cur_offset); + offset += static_cast<int>(MessageBuffer::PointerDescriptor::GetDataSize() / sizeof(u32)); + + // Extract address/size. + const uint64_t src_pointer = src_desc.GetAddress(); + const size_t recv_size = src_desc.GetSize(); + uint64_t recv_pointer = 0; + + // Process the buffer, if it has a size. + if (recv_size > 0) { + // If using indexing, set index. + if (dst_recv_list.IsIndex()) { + pointer_key = src_desc.GetIndex(); } + + // Get the buffer. + dst_recv_list.GetBuffer(recv_pointer, recv_size, pointer_key); + R_UNLESS(recv_pointer != 0, ResultOutOfResource); + + // Perform the pointer data copy. + const bool dst_heap = dst_user && dst_recv_list.IsToMessageBuffer(); + const auto dst_state = + dst_heap ? KMemoryState::FlagReferenceCounted : KMemoryState::FlagLinearMapped; + const KMemoryPermission dst_perm = + dst_heap ? KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite + : KMemoryPermission::UserReadWrite; + R_TRY(dst_page_table.CopyMemoryFromUserToLinear( + recv_pointer, recv_size, dst_state, dst_state, dst_perm, KMemoryAttribute::Uncached, + KMemoryAttribute::None, src_pointer)); } - // Notify. - this->NotifyAvailable(ResultSessionClosed); + // Set the output descriptor. + dst_msg.Set(cur_offset, MessageBuffer::PointerDescriptor(reinterpret_cast<void*>(recv_pointer), + recv_size, src_desc.GetIndex())); + + R_SUCCEED(); } -bool KServerSession::IsSignaled() const { - ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); +Result SendMessage(KernelCore& kernel, uint64_t src_message_buffer, size_t src_buffer_size, + KPhysicalAddress src_message_paddr, KThread& dst_thread, + uint64_t dst_message_buffer, size_t dst_buffer_size, KServerSession* session, + KSessionRequest* request) { + // Prepare variables for send. + KThread& src_thread = GetCurrentThread(kernel); + KProcess& dst_process = *(dst_thread.GetOwnerProcess()); + KProcess& src_process = *(src_thread.GetOwnerProcess()); + auto& dst_page_table = dst_process.GetPageTable(); + auto& src_page_table = src_process.GetPageTable(); + + // NOTE: Session is used only for debugging, and so may go unused. + (void)session; + + // Determine the message buffers. + u32 *dst_msg_ptr, *src_msg_ptr; + bool dst_user, src_user; + + if (dst_message_buffer) { + // NOTE: Nintendo does not check the result of this GetPhysicalAddress call. + dst_msg_ptr = dst_page_table.GetMemory().GetPointer<u32>(dst_message_buffer); + dst_user = true; + } else { + dst_msg_ptr = dst_page_table.GetMemory().GetPointer<u32>(dst_thread.GetTlsAddress()); + dst_buffer_size = MessageBufferSize; + dst_message_buffer = GetInteger(dst_thread.GetTlsAddress()); + dst_user = false; + } - // If the client is closed, we're always signaled. - if (m_parent->IsClientClosed()) { - return true; + if (src_message_buffer) { + src_msg_ptr = src_page_table.GetMemory().GetPointer<u32>(src_message_buffer); + src_user = true; + } else { + src_msg_ptr = src_page_table.GetMemory().GetPointer<u32>(src_thread.GetTlsAddress()); + src_buffer_size = MessageBufferSize; + src_message_buffer = GetInteger(src_thread.GetTlsAddress()); + src_user = false; } - // Otherwise, we're signaled if we have a request and aren't handling one. - return !m_request_list.empty() && m_current_request == nullptr; + // Parse the headers. + const MessageBuffer dst_msg(dst_msg_ptr, dst_buffer_size); + const MessageBuffer src_msg(src_msg_ptr, src_buffer_size); + const MessageBuffer::MessageHeader dst_header(dst_msg); + const MessageBuffer::MessageHeader src_header(src_msg); + const MessageBuffer::SpecialHeader dst_special_header(dst_msg, dst_header); + const MessageBuffer::SpecialHeader src_special_header(src_msg, src_header); + + // Get the end of the source message. + const size_t src_end_offset = + MessageBuffer::GetRawDataIndex(src_header, src_special_header) + src_header.GetRawCount(); + + // Declare variables for processing. + int offset = 0; + int pointer_key = 0; + bool processed_special_data = false; + + // Send the message. + { + // Make sure that we end up in a clean state on error. + ON_RESULT_FAILURE { + // Cleanup special data. + if (processed_special_data) { + if (src_header.GetHasSpecialHeader()) { + CleanupSpecialData(dst_process, dst_msg_ptr, dst_buffer_size); + } + } else { + CleanupServerHandles(kernel, src_user ? src_message_buffer : 0, src_buffer_size, + src_message_paddr); + } + + // Cleanup mappings. + CleanupMap(request, std::addressof(src_process), std::addressof(dst_page_table)); + }; + + // Ensure that the headers fit. + R_UNLESS(MessageBuffer::GetMessageBufferSize(src_header, src_special_header) <= + src_buffer_size, + ResultInvalidCombination); + R_UNLESS(MessageBuffer::GetMessageBufferSize(dst_header, dst_special_header) <= + dst_buffer_size, + ResultInvalidCombination); + + // Ensure the receive list offset is after the end of raw data. + if (dst_header.GetReceiveListOffset()) { + R_UNLESS(dst_header.GetReceiveListOffset() >= + MessageBuffer::GetRawDataIndex(dst_header, dst_special_header) + + dst_header.GetRawCount(), + ResultInvalidCombination); + } + + // Ensure that the destination buffer is big enough to receive the source. + R_UNLESS(dst_buffer_size >= src_end_offset * sizeof(u32), ResultMessageTooLarge); + + // Replies must have no buffers. + R_UNLESS(src_header.GetSendCount() == 0, ResultInvalidCombination); + R_UNLESS(src_header.GetReceiveCount() == 0, ResultInvalidCombination); + R_UNLESS(src_header.GetExchangeCount() == 0, ResultInvalidCombination); + + // Get the receive list. + const s32 dst_recv_list_idx = + MessageBuffer::GetReceiveListIndex(dst_header, dst_special_header); + ReceiveList dst_recv_list(dst_msg_ptr, dst_message_buffer, dst_page_table, dst_header, + dst_special_header, dst_buffer_size, src_end_offset, + dst_recv_list_idx, !dst_user); + + // Handle any receive buffers. + for (size_t i = 0; i < request->GetReceiveCount(); ++i) { + R_TRY(ProcessSendMessageReceiveMapping( + src_page_table, dst_page_table, request->GetReceiveClientAddress(i), + request->GetReceiveServerAddress(i), request->GetReceiveSize(i), + request->GetReceiveMemoryState(i))); + } + + // Handle any exchange buffers. + for (size_t i = 0; i < request->GetExchangeCount(); ++i) { + R_TRY(ProcessSendMessageReceiveMapping( + src_page_table, dst_page_table, request->GetExchangeClientAddress(i), + request->GetExchangeServerAddress(i), request->GetExchangeSize(i), + request->GetExchangeMemoryState(i))); + } + + // Set the header. + offset = dst_msg.Set(src_header); + + // Process any special data. + ASSERT(GetCurrentThreadPointer(kernel) == std::addressof(src_thread)); + processed_special_data = true; + if (src_header.GetHasSpecialHeader()) { + R_TRY(ProcessMessageSpecialData<true>(offset, dst_process, src_process, src_thread, + dst_msg, src_msg, src_special_header)); + } + + // Process any pointer buffers. + for (auto i = 0; i < src_header.GetPointerCount(); ++i) { + R_TRY(ProcessSendMessagePointerDescriptors( + offset, pointer_key, src_page_table, dst_page_table, dst_msg, src_msg, + dst_recv_list, + dst_user && + dst_header.GetReceiveListCount() == + MessageBuffer::MessageHeader::ReceiveListCountType_ToMessageBuffer)); + } + + // Clear any map alias buffers. + for (auto i = 0; i < src_header.GetMapAliasCount(); ++i) { + offset = dst_msg.Set(offset, MessageBuffer::MapAliasDescriptor()); + } + + // Process any raw data. + if (const auto raw_count = src_header.GetRawCount(); raw_count != 0) { + // Get the offset and size. + const size_t offset_words = offset * sizeof(u32); + const size_t raw_size = raw_count * sizeof(u32); + + if (!dst_user && !src_user) { + // Fast case is TLS -> TLS, do raw memcpy if we can. + std::memcpy(dst_msg_ptr + offset, src_msg_ptr + offset, raw_size); + } else if (src_user) { + // Determine how much fast size we can copy. + const size_t max_fast_size = std::min<size_t>(offset_words + raw_size, PageSize); + const size_t fast_size = max_fast_size - offset_words; + + // Determine dst state; if user buffer, we require heap, and otherwise only linear + // mapped (to enable tls use). + const auto dst_state = + dst_user ? KMemoryState::FlagReferenceCounted : KMemoryState::FlagLinearMapped; + + // Determine the dst permission. User buffer should be unmapped + read, TLS should + // be user readable. + const KMemoryPermission dst_perm = + dst_user ? KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite + : KMemoryPermission::UserReadWrite; + + // Perform the fast part of the copy. + R_TRY(dst_page_table.CopyMemoryFromKernelToLinear( + dst_message_buffer + offset_words, fast_size, dst_state, dst_state, dst_perm, + KMemoryAttribute::Uncached, KMemoryAttribute::None, src_msg_ptr + offset)); + + // If the fast part of the copy didn't get everything, perform the slow part of the + // copy. + if (fast_size < raw_size) { + R_TRY(dst_page_table.CopyMemoryFromHeapToHeap( + dst_page_table, dst_message_buffer + max_fast_size, raw_size - fast_size, + dst_state, dst_state, dst_perm, KMemoryAttribute::Uncached, + KMemoryAttribute::None, src_message_buffer + max_fast_size, + KMemoryState::FlagReferenceCounted, KMemoryState::FlagReferenceCounted, + KMemoryPermission::NotMapped | KMemoryPermission::KernelRead, + KMemoryAttribute::Uncached | KMemoryAttribute::Locked, + KMemoryAttribute::Locked)); + } + } else /* if (dst_user) */ { + // The destination is a user buffer, so it should be unmapped + readable. + constexpr KMemoryPermission DestinationPermission = + KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite; + + // Copy the memory. + R_TRY(dst_page_table.CopyMemoryFromUserToLinear( + dst_message_buffer + offset_words, raw_size, KMemoryState::FlagReferenceCounted, + KMemoryState::FlagReferenceCounted, DestinationPermission, + KMemoryAttribute::Uncached, KMemoryAttribute::None, + src_message_buffer + offset_words)); + } + } + } + + // Perform (and validate) any remaining cleanup. + R_RETURN(CleanupMap(request, std::addressof(src_process), std::addressof(dst_page_table))); } -Result KServerSession::OnRequest(KSessionRequest* request) { - // Create the wait queue. - ThreadQueueImplForKServerSessionRequest wait_queue{m_kernel}; +void ReplyAsyncError(KProcess* to_process, uint64_t to_msg_buf, size_t to_msg_buf_size, + Result result) { + // Convert the address to a linear pointer. + u32* to_msg = to_process->GetMemory().GetPointer<u32>(to_msg_buf); + + // Set the error. + MessageBuffer msg(to_msg, to_msg_buf_size); + msg.SetAsyncResult(result); +} + +} // namespace + +KServerSession::KServerSession(KernelCore& kernel) + : KSynchronizationObject{kernel}, m_lock{m_kernel} {} + +KServerSession::~KServerSession() = default; + +void KServerSession::Destroy() { + m_parent->OnServerClosed(); + + this->CleanupRequests(); + + m_parent->Close(); +} + +Result KServerSession::ReceiveRequest(uintptr_t server_message, uintptr_t server_buffer_size, + KPhysicalAddress server_message_paddr, + std::shared_ptr<Service::HLERequestContext>* out_context, + std::weak_ptr<Service::SessionRequestManager> manager) { + // Lock the session. + KScopedLightLock lk{m_lock}; + + // Get the request and client thread. + KSessionRequest* request; + KThread* client_thread; { - // Lock the scheduler. KScopedSchedulerLock sl{m_kernel}; - // Ensure that we can handle new requests. - R_UNLESS(!m_parent->IsServerClosed(), ResultSessionClosed); + // Ensure that we can service the request. + R_UNLESS(!m_parent->IsClientClosed(), ResultSessionClosed); - // Check that we're not terminating. - R_UNLESS(!GetCurrentThread(m_kernel).IsTerminationRequested(), ResultTerminationRequested); + // Ensure we aren't already servicing a request. + R_UNLESS(m_current_request == nullptr, ResultNotFound); - // Get whether we're empty. - const bool was_empty = m_request_list.empty(); + // Ensure we have a request to service. + R_UNLESS(!m_request_list.empty(), ResultNotFound); - // Add the request to the list. - request->Open(); - m_request_list.push_back(*request); + // Pop the first request from the list. + request = std::addressof(m_request_list.front()); + m_request_list.pop_front(); - // If we were empty, signal. - if (was_empty) { - this->NotifyAvailable(); + // Get the thread for the request. + client_thread = request->GetThread(); + R_UNLESS(client_thread != nullptr, ResultSessionClosed); + + // Open the client thread. + client_thread->Open(); + } + + SCOPE_EXIT({ client_thread->Close(); }); + + // Set the request as our current. + m_current_request = request; + + // Get the client address. + uint64_t client_message = request->GetAddress(); + size_t client_buffer_size = request->GetSize(); + bool recv_list_broken = false; + + // Receive the message. + Result result = ResultSuccess; + + if (out_context != nullptr) { + // HLE request. + if (!client_message) { + client_message = GetInteger(client_thread->GetTlsAddress()); } + Core::Memory::Memory& memory{client_thread->GetOwnerProcess()->GetMemory()}; + u32* cmd_buf{reinterpret_cast<u32*>(memory.GetPointer(client_message))}; + *out_context = + std::make_shared<Service::HLERequestContext>(m_kernel, memory, this, client_thread); + (*out_context)->SetSessionRequestManager(manager); + (*out_context)->PopulateFromIncomingCommandBuffer(cmd_buf); + // We succeeded. + R_SUCCEED(); + } else { + result = ReceiveMessage(m_kernel, recv_list_broken, server_message, server_buffer_size, + server_message_paddr, *client_thread, client_message, + client_buffer_size, this, request); + } - // If we have a request event, this is asynchronous, and we don't need to wait. - R_SUCCEED_IF(request->GetEvent() != nullptr); + // Handle cleanup on receive failure. + if (R_FAILED(result)) { + // Cache the result to return it to the client. + const Result result_for_client = result; - // This is a synchronous request, so we should wait for our request to complete. - GetCurrentThread(m_kernel).SetWaitReasonForDebugging(ThreadWaitReasonForDebugging::IPC); - GetCurrentThread(m_kernel).BeginWait(std::addressof(wait_queue)); + // Clear the current request. + { + KScopedSchedulerLock sl(m_kernel); + ASSERT(m_current_request == request); + m_current_request = nullptr; + if (!m_request_list.empty()) { + this->NotifyAvailable(); + } + } + + // Reply to the client. + { + // After we reply, close our reference to the request. + SCOPE_EXIT({ request->Close(); }); + + // Get the event to check whether the request is async. + if (KEvent* event = request->GetEvent(); event != nullptr) { + // The client sent an async request. + KProcess* client = client_thread->GetOwnerProcess(); + auto& client_pt = client->GetPageTable(); + + // Send the async result. + if (R_FAILED(result_for_client)) { + ReplyAsyncError(client, client_message, client_buffer_size, result_for_client); + } + + // Unlock the client buffer. + // NOTE: Nintendo does not check the result of this. + client_pt.UnlockForIpcUserBuffer(client_message, client_buffer_size); + + // Signal the event. + event->Signal(); + } else { + // End the client thread's wait. + KScopedSchedulerLock sl(m_kernel); + + if (!client_thread->IsTerminationRequested()) { + client_thread->EndWait(result_for_client); + } + } + } + + // Set the server result. + if (recv_list_broken) { + result = ResultReceiveListBroken; + } else { + result = ResultNotFound; + } } - return GetCurrentThread(m_kernel).GetWaitResult(); + R_RETURN(result); } -Result KServerSession::SendReply(bool is_hle) { +Result KServerSession::SendReply(uintptr_t server_message, uintptr_t server_buffer_size, + KPhysicalAddress server_message_paddr, bool is_hle) { // Lock the session. KScopedLightLock lk{m_lock}; @@ -327,7 +1239,7 @@ Result KServerSession::SendReply(bool is_hle) { SCOPE_EXIT({ request->Close(); }); // Extract relevant information from the request. - const uintptr_t client_message = request->GetAddress(); + const uint64_t client_message = request->GetAddress(); const size_t client_buffer_size = request->GetSize(); KThread* client_thread = request->GetThread(); KEvent* event = request->GetEvent(); @@ -342,31 +1254,28 @@ Result KServerSession::SendReply(bool is_hle) { // HLE servers write directly to a pointer to the thread command buffer. Therefore // the reply has already been written in this case. } else { - Core::Memory::Memory& memory{client_thread->GetOwnerProcess()->GetMemory()}; - KThread* server_thread = GetCurrentThreadPointer(m_kernel); - KProcess& src_process = *client_thread->GetOwnerProcess(); - KProcess& dst_process = *server_thread->GetOwnerProcess(); - UNIMPLEMENTED_IF(server_thread->GetOwnerProcess() != client_thread->GetOwnerProcess()); - - auto* src_msg_buffer = memory.GetPointer<u32>(server_thread->GetTlsAddress()); - auto* dst_msg_buffer = memory.GetPointer<u32>(client_message); - std::memcpy(dst_msg_buffer, src_msg_buffer, client_buffer_size); - - // Translate special header ad-hoc. - MessageBuffer src_msg(src_msg_buffer, client_buffer_size); - MessageBuffer::MessageHeader src_header(src_msg); - MessageBuffer::SpecialHeader src_special_header(src_msg, src_header); - if (src_header.GetHasSpecialHeader()) { - MessageBuffer dst_msg(dst_msg_buffer, client_buffer_size); - result = ProcessMessageSpecialData<true>(dst_process, src_process, *server_thread, - dst_msg, src_msg, src_special_header); - if (R_FAILED(result)) { - CleanupSpecialData(dst_process, dst_msg_buffer, client_buffer_size); - } - } + result = SendMessage(m_kernel, server_message, server_buffer_size, server_message_paddr, + *client_thread, client_message, client_buffer_size, this, request); + } + } else if (!is_hle) { + // Otherwise, we'll need to do some cleanup. + KProcess* server_process = request->GetServerProcess(); + KProcess* client_process = + (client_thread != nullptr) ? client_thread->GetOwnerProcess() : nullptr; + KProcessPageTable* client_page_table = + (client_process != nullptr) ? std::addressof(client_process->GetPageTable()) : nullptr; + + // Cleanup server handles. + result = CleanupServerHandles(m_kernel, server_message, server_buffer_size, + server_message_paddr); + + // Cleanup mappings. + Result cleanup_map_result = CleanupMap(request, server_process, client_page_table); + + // If we successfully cleaned up handles, use the map cleanup result as our result. + if (R_SUCCEEDED(result)) { + result = cleanup_map_result; } - } else { - result = ResultSessionClosed; } // Select a result for the client. @@ -381,19 +1290,18 @@ Result KServerSession::SendReply(bool is_hle) { // If there's a client thread, update it. if (client_thread != nullptr) { if (event != nullptr) { - // // Get the client process/page table. - // KProcess *client_process = client_thread->GetOwnerProcess(); - // KProcessPageTable *client_page_table = std::addressof(client_process->PageTable()); + // Get the client process/page table. + KProcess* client_process = client_thread->GetOwnerProcess(); + KProcessPageTable* client_page_table = std::addressof(client_process->GetPageTable()); - // // If we need to, reply with an async error. - // if (R_FAILED(client_result)) { - // ReplyAsyncError(client_process, client_message, client_buffer_size, - // client_result); - // } + // If we need to, reply with an async error. + if (R_FAILED(client_result)) { + ReplyAsyncError(client_process, client_message, client_buffer_size, client_result); + } - // // Unlock the client buffer. - // // NOTE: Nintendo does not check the result of this. - // client_page_table->UnlockForIpcUserBuffer(client_message, client_buffer_size); + // Unlock the client buffer. + // NOTE: Nintendo does not check the result of this. + client_page_table->UnlockForIpcUserBuffer(client_message, client_buffer_size); // Signal the event. event->Signal(); @@ -410,91 +1318,53 @@ Result KServerSession::SendReply(bool is_hle) { R_RETURN(result); } -Result KServerSession::ReceiveRequest(std::shared_ptr<Service::HLERequestContext>* out_context, - std::weak_ptr<Service::SessionRequestManager> manager) { - // Lock the session. - KScopedLightLock lk{m_lock}; - - // Get the request and client thread. - KSessionRequest* request; - KThread* client_thread; +Result KServerSession::OnRequest(KSessionRequest* request) { + // Create the wait queue. + ThreadQueueImplForKServerSessionRequest wait_queue{m_kernel}; { + // Lock the scheduler. KScopedSchedulerLock sl{m_kernel}; - // Ensure that we can service the request. - R_UNLESS(!m_parent->IsClientClosed(), ResultSessionClosed); - - // Ensure we aren't already servicing a request. - R_UNLESS(m_current_request == nullptr, ResultNotFound); + // Ensure that we can handle new requests. + R_UNLESS(!m_parent->IsServerClosed(), ResultSessionClosed); - // Ensure we have a request to service. - R_UNLESS(!m_request_list.empty(), ResultNotFound); + // Check that we're not terminating. + R_UNLESS(!GetCurrentThread(m_kernel).IsTerminationRequested(), ResultTerminationRequested); - // Pop the first request from the list. - request = std::addressof(m_request_list.front()); - m_request_list.pop_front(); + // Get whether we're empty. + const bool was_empty = m_request_list.empty(); - // Get the thread for the request. - client_thread = request->GetThread(); - R_UNLESS(client_thread != nullptr, ResultSessionClosed); + // Add the request to the list. + request->Open(); + m_request_list.push_back(*request); - // Open the client thread. - client_thread->Open(); - } + // If we were empty, signal. + if (was_empty) { + this->NotifyAvailable(); + } - SCOPE_EXIT({ client_thread->Close(); }); + // If we have a request event, this is asynchronous, and we don't need to wait. + R_SUCCEED_IF(request->GetEvent() != nullptr); - // Set the request as our current. - m_current_request = request; + // This is a synchronous request, so we should wait for our request to complete. + GetCurrentThread(m_kernel).SetWaitReasonForDebugging(ThreadWaitReasonForDebugging::IPC); + GetCurrentThread(m_kernel).BeginWait(std::addressof(wait_queue)); + } - // Get the client address. - uintptr_t client_message = request->GetAddress(); - size_t client_buffer_size = request->GetSize(); - // bool recv_list_broken = false; + return GetCurrentThread(m_kernel).GetWaitResult(); +} - if (!client_message) { - client_message = GetInteger(client_thread->GetTlsAddress()); - client_buffer_size = MessageBufferSize; - } +bool KServerSession::IsSignaled() const { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); - // Receive the message. - Core::Memory::Memory& memory{client_thread->GetOwnerProcess()->GetMemory()}; - if (out_context != nullptr) { - // HLE request. - u32* cmd_buf{reinterpret_cast<u32*>(memory.GetPointer(client_message))}; - *out_context = - std::make_shared<Service::HLERequestContext>(m_kernel, memory, this, client_thread); - (*out_context)->SetSessionRequestManager(manager); - (*out_context) - ->PopulateFromIncomingCommandBuffer(*client_thread->GetOwnerProcess(), cmd_buf); - } else { - KThread* server_thread = GetCurrentThreadPointer(m_kernel); - KProcess& src_process = *client_thread->GetOwnerProcess(); - KProcess& dst_process = *server_thread->GetOwnerProcess(); - UNIMPLEMENTED_IF(client_thread->GetOwnerProcess() != server_thread->GetOwnerProcess()); - - auto* src_msg_buffer = memory.GetPointer<u32>(client_message); - auto* dst_msg_buffer = memory.GetPointer<u32>(server_thread->GetTlsAddress()); - std::memcpy(dst_msg_buffer, src_msg_buffer, client_buffer_size); - - // Translate special header ad-hoc. - // TODO: fix this mess - MessageBuffer src_msg(src_msg_buffer, client_buffer_size); - MessageBuffer::MessageHeader src_header(src_msg); - MessageBuffer::SpecialHeader src_special_header(src_msg, src_header); - if (src_header.GetHasSpecialHeader()) { - MessageBuffer dst_msg(dst_msg_buffer, client_buffer_size); - Result res = ProcessMessageSpecialData<false>(dst_process, src_process, *client_thread, - dst_msg, src_msg, src_special_header); - if (R_FAILED(res)) { - CleanupSpecialData(dst_process, dst_msg_buffer, client_buffer_size); - } - } + // If the client is closed, we're always signaled. + if (m_parent->IsClientClosed()) { + return true; } - // We succeeded. - R_SUCCEED(); + // Otherwise, we're signaled if we have a request and aren't handling one. + return !m_request_list.empty() && m_current_request == nullptr; } void KServerSession::CleanupRequests() { @@ -527,31 +1397,30 @@ void KServerSession::CleanupRequests() { SCOPE_EXIT({ request->Close(); }); // Extract relevant information from the request. - // const uintptr_t client_message = request->GetAddress(); - // const size_t client_buffer_size = request->GetSize(); + const uint64_t client_message = request->GetAddress(); + const size_t client_buffer_size = request->GetSize(); KThread* client_thread = request->GetThread(); KEvent* event = request->GetEvent(); - // KProcess *server_process = request->GetServerProcess(); - // KProcess *client_process = (client_thread != nullptr) ? - // client_thread->GetOwnerProcess() : nullptr; - // KProcessPageTable *client_page_table = (client_process != nullptr) ? - // std::addressof(client_process->GetPageTable()) - // : nullptr; + KProcess* server_process = request->GetServerProcess(); + KProcess* client_process = + (client_thread != nullptr) ? client_thread->GetOwnerProcess() : nullptr; + KProcessPageTable* client_page_table = + (client_process != nullptr) ? std::addressof(client_process->GetPageTable()) : nullptr; // Cleanup the mappings. - // Result result = CleanupMap(request, server_process, client_page_table); + Result result = CleanupMap(request, server_process, client_page_table); // If there's a client thread, update it. if (client_thread != nullptr) { if (event != nullptr) { - // // We need to reply async. - // ReplyAsyncError(client_process, client_message, client_buffer_size, - // (R_SUCCEEDED(result) ? ResultSessionClosed : result)); + // We need to reply async. + ReplyAsyncError(client_process, client_message, client_buffer_size, + (R_SUCCEEDED(result) ? ResultSessionClosed : result)); - // // Unlock the client buffer. + // Unlock the client buffer. // NOTE: Nintendo does not check the result of this. - // client_page_table->UnlockForIpcUserBuffer(client_message, client_buffer_size); + client_page_table->UnlockForIpcUserBuffer(client_message, client_buffer_size); // Signal the event. event->Signal(); @@ -567,4 +1436,97 @@ void KServerSession::CleanupRequests() { } } +void KServerSession::OnClientClosed() { + KScopedLightLock lk{m_lock}; + + // Handle any pending requests. + KSessionRequest* prev_request = nullptr; + while (true) { + // Declare variables for processing the request. + KSessionRequest* request = nullptr; + KEvent* event = nullptr; + KThread* thread = nullptr; + bool cur_request = false; + bool terminate = false; + + // Get the next request. + { + KScopedSchedulerLock sl{m_kernel}; + + if (m_current_request != nullptr && m_current_request != prev_request) { + // Set the request, open a reference as we process it. + request = m_current_request; + request->Open(); + cur_request = true; + + // Get thread and event for the request. + thread = request->GetThread(); + event = request->GetEvent(); + + // If the thread is terminating, handle that. + if (thread->IsTerminationRequested()) { + request->ClearThread(); + request->ClearEvent(); + terminate = true; + } + + prev_request = request; + } else if (!m_request_list.empty()) { + // Pop the request from the front of the list. + request = std::addressof(m_request_list.front()); + m_request_list.pop_front(); + + // Get thread and event for the request. + thread = request->GetThread(); + event = request->GetEvent(); + } + } + + // If there are no requests, we're done. + if (request == nullptr) { + break; + } + + // All requests must have threads. + ASSERT(thread != nullptr); + + // Ensure that we close the request when done. + SCOPE_EXIT({ request->Close(); }); + + // If we're terminating, close a reference to the thread and event. + if (terminate) { + thread->Close(); + if (event != nullptr) { + event->Close(); + } + } + + // If we need to, reply. + if (event != nullptr && !cur_request) { + // There must be no mappings. + ASSERT(request->GetSendCount() == 0); + ASSERT(request->GetReceiveCount() == 0); + ASSERT(request->GetExchangeCount() == 0); + + // Get the process and page table. + KProcess* client_process = thread->GetOwnerProcess(); + auto& client_pt = client_process->GetPageTable(); + + // Reply to the request. + ReplyAsyncError(client_process, request->GetAddress(), request->GetSize(), + ResultSessionClosed); + + // Unlock the buffer. + // NOTE: Nintendo does not check the result of this. + client_pt.UnlockForIpcUserBuffer(request->GetAddress(), request->GetSize()); + + // Signal the event. + event->Signal(); + } + } + + // Notify. + this->NotifyAvailable(ResultSessionClosed); +} + } // namespace Kernel diff --git a/src/core/hle/kernel/k_server_session.h b/src/core/hle/kernel/k_server_session.h index 403891919..2876c231b 100644 --- a/src/core/hle/kernel/k_server_session.h +++ b/src/core/hle/kernel/k_server_session.h @@ -49,14 +49,21 @@ public: bool IsSignaled() const override; void OnClientClosed(); - /// TODO: flesh these out to match the real kernel Result OnRequest(KSessionRequest* request); - Result SendReply(bool is_hle = false); - Result ReceiveRequest(std::shared_ptr<Service::HLERequestContext>* out_context = nullptr, + Result SendReply(uintptr_t server_message, uintptr_t server_buffer_size, + KPhysicalAddress server_message_paddr, bool is_hle = false); + Result ReceiveRequest(uintptr_t server_message, uintptr_t server_buffer_size, + KPhysicalAddress server_message_paddr, + std::shared_ptr<Service::HLERequestContext>* out_context = nullptr, std::weak_ptr<Service::SessionRequestManager> manager = {}); Result SendReplyHLE() { - return SendReply(true); + R_RETURN(this->SendReply(0, 0, 0, true)); + } + + Result ReceiveRequestHLE(std::shared_ptr<Service::HLERequestContext>* out_context, + std::weak_ptr<Service::SessionRequestManager> manager) { + R_RETURN(this->ReceiveRequest(0, 0, 0, out_context, manager)); } private: diff --git a/src/core/hle/kernel/k_session.cpp b/src/core/hle/kernel/k_session.cpp index 44d7a8f02..4a1f6027e 100644 --- a/src/core/hle/kernel/k_session.cpp +++ b/src/core/hle/kernel/k_session.cpp @@ -33,8 +33,7 @@ void KSession::Initialize(KClientPort* client_port, uintptr_t name) { m_name = name; // Set our owner process. - //! FIXME: this is the wrong process! - m_process = m_kernel.ApplicationProcess(); + m_process = GetCurrentProcessPointer(m_kernel); m_process->Open(); // Set our port. diff --git a/src/core/hle/kernel/k_thread.cpp b/src/core/hle/kernel/k_thread.cpp index 7d9a6e9cf..24394d222 100644 --- a/src/core/hle/kernel/k_thread.cpp +++ b/src/core/hle/kernel/k_thread.cpp @@ -1422,8 +1422,7 @@ s32 GetCurrentCoreId(KernelCore& kernel) { } Core::Memory::Memory& GetCurrentMemory(KernelCore& kernel) { - // TODO: per-process memory - return kernel.System().ApplicationMemory(); + return GetCurrentProcess(kernel).GetMemory(); } KScopedDisableDispatch::~KScopedDisableDispatch() { diff --git a/src/core/hle/kernel/k_thread.h b/src/core/hle/kernel/k_thread.h index e9925d231..f13e232b2 100644 --- a/src/core/hle/kernel/k_thread.h +++ b/src/core/hle/kernel/k_thread.h @@ -314,11 +314,7 @@ public: m_current_core_id = core; } - KProcess* GetOwnerProcess() { - return m_parent; - } - - const KProcess* GetOwnerProcess() const { + KProcess* GetOwnerProcess() const { return m_parent; } diff --git a/src/core/hle/kernel/k_transfer_memory.h b/src/core/hle/kernel/k_transfer_memory.h index 8a0b08761..530b45218 100644 --- a/src/core/hle/kernel/k_transfer_memory.h +++ b/src/core/hle/kernel/k_transfer_memory.h @@ -5,6 +5,7 @@ #include <optional> +#include "core/hle/kernel/k_light_lock.h" #include "core/hle/kernel/k_page_group.h" #include "core/hle/kernel/slab_helpers.h" #include "core/hle/kernel/svc_types.h" diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index b515f6a18..1030f0c12 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -68,8 +68,6 @@ struct KernelCore::Impl { global_object_list_container = std::make_unique<KAutoObjectWithListContainer>(kernel); global_scheduler_context = std::make_unique<Kernel::GlobalSchedulerContext>(kernel); - global_handle_table = std::make_unique<Kernel::KHandleTable>(kernel); - global_handle_table->Initialize(KHandleTable::MaxTableSize); is_phantom_mode_for_singlecore = false; @@ -121,13 +119,8 @@ struct KernelCore::Impl { next_user_process_id = KProcess::ProcessIdMin; next_thread_id = 1; - global_handle_table->Finalize(); - global_handle_table.reset(); - preemption_event = nullptr; - exclusive_monitor.reset(); - // Cleanup persistent kernel objects auto CleanupObject = [](KAutoObject* obj) { if (obj) { @@ -191,8 +184,6 @@ struct KernelCore::Impl { } void InitializePhysicalCores() { - exclusive_monitor = - Core::MakeExclusiveMonitor(system.ApplicationMemory(), Core::Hardware::NUM_CPU_CORES); for (u32 i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { const s32 core{static_cast<s32>(i)}; @@ -791,10 +782,6 @@ struct KernelCore::Impl { std::shared_ptr<Core::Timing::EventType> preemption_event; - // This is the kernel's handle table or supervisor handle table which - // stores all the objects in place. - std::unique_ptr<KHandleTable> global_handle_table; - std::unique_ptr<KAutoObjectWithListContainer> global_object_list_container; std::unique_ptr<KObjectNameGlobalData> object_name_global_data; @@ -805,7 +792,6 @@ struct KernelCore::Impl { std::mutex server_lock; std::vector<std::unique_ptr<Service::ServerManager>> server_managers; - std::unique_ptr<Core::ExclusiveMonitor> exclusive_monitor; std::array<std::unique_ptr<Kernel::PhysicalCore>, Core::Hardware::NUM_CPU_CORES> cores; // Next host thead ID to use, 0-3 IDs represent core threads, >3 represent others @@ -882,10 +868,6 @@ KResourceLimit* KernelCore::GetSystemResourceLimit() { return impl->system_resource_limit; } -KScopedAutoObject<KThread> KernelCore::RetrieveThreadFromGlobalHandleTable(Handle handle) const { - return impl->global_handle_table->GetObject<KThread>(handle); -} - void KernelCore::AppendNewProcess(KProcess* process) { impl->process_list.push_back(process); } @@ -959,14 +941,6 @@ Kernel::KHardwareTimer& KernelCore::HardwareTimer() { return *impl->hardware_timer; } -Core::ExclusiveMonitor& KernelCore::GetExclusiveMonitor() { - return *impl->exclusive_monitor; -} - -const Core::ExclusiveMonitor& KernelCore::GetExclusiveMonitor() const { - return *impl->exclusive_monitor; -} - KAutoObjectWithListContainer& KernelCore::ObjectListContainer() { return *impl->global_object_list_container; } @@ -1030,14 +1004,6 @@ u64 KernelCore::CreateNewUserProcessID() { return impl->next_user_process_id++; } -KHandleTable& KernelCore::GlobalHandleTable() { - return *impl->global_handle_table; -} - -const KHandleTable& KernelCore::GlobalHandleTable() const { - return *impl->global_handle_table; -} - void KernelCore::RegisterCoreThread(std::size_t core_id) { impl->RegisterCoreThread(core_id); } diff --git a/src/core/hle/kernel/kernel.h b/src/core/hle/kernel/kernel.h index 78c88902c..5d4102145 100644 --- a/src/core/hle/kernel/kernel.h +++ b/src/core/hle/kernel/kernel.h @@ -116,9 +116,6 @@ public: /// Retrieves a shared pointer to the system resource limit instance. KResourceLimit* GetSystemResourceLimit(); - /// Retrieves a shared pointer to a Thread instance within the thread wakeup handle table. - KScopedAutoObject<KThread> RetrieveThreadFromGlobalHandleTable(Handle handle) const; - /// Adds the given shared pointer to an internal list of active processes. void AppendNewProcess(KProcess* process); @@ -170,10 +167,6 @@ public: /// Stops execution of 'id' core, in order to reschedule a new thread. void PrepareReschedule(std::size_t id); - Core::ExclusiveMonitor& GetExclusiveMonitor(); - - const Core::ExclusiveMonitor& GetExclusiveMonitor() const; - KAutoObjectWithListContainer& ObjectListContainer(); const KAutoObjectWithListContainer& ObjectListContainer() const; diff --git a/src/core/hle/kernel/message_buffer.h b/src/core/hle/kernel/message_buffer.h index 75b275310..d528a9bb3 100644 --- a/src/core/hle/kernel/message_buffer.h +++ b/src/core/hle/kernel/message_buffer.h @@ -18,13 +18,13 @@ public: static constexpr inline u64 NullTag = 0; public: - enum class ReceiveListCountType : u32 { - None = 0, - ToMessageBuffer = 1, - ToSingleBuffer = 2, + enum ReceiveListCountType : u32 { + ReceiveListCountType_None = 0, + ReceiveListCountType_ToMessageBuffer = 1, + ReceiveListCountType_ToSingleBuffer = 2, - CountOffset = 2, - CountMax = 13, + ReceiveListCountType_CountOffset = 2, + ReceiveListCountType_CountMax = 13, }; private: @@ -591,16 +591,16 @@ public: // Add the size of the receive list. const auto count = hdr.GetReceiveListCount(); switch (count) { - case MessageHeader::ReceiveListCountType::None: + case MessageHeader::ReceiveListCountType_None: break; - case MessageHeader::ReceiveListCountType::ToMessageBuffer: + case MessageHeader::ReceiveListCountType_ToMessageBuffer: break; - case MessageHeader::ReceiveListCountType::ToSingleBuffer: + case MessageHeader::ReceiveListCountType_ToSingleBuffer: msg_size += ReceiveListEntry::GetDataSize(); break; default: msg_size += (static_cast<s32>(count) - - static_cast<s32>(MessageHeader::ReceiveListCountType::CountOffset)) * + static_cast<s32>(MessageHeader::ReceiveListCountType_CountOffset)) * ReceiveListEntry::GetDataSize(); break; } diff --git a/src/core/hle/kernel/svc/svc_info.cpp b/src/core/hle/kernel/svc/svc_info.cpp index ada998772..231e4d0e1 100644 --- a/src/core/hle/kernel/svc/svc_info.cpp +++ b/src/core/hle/kernel/svc/svc_info.cpp @@ -118,7 +118,6 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle R_SUCCEED(); case InfoType::IsApplication: - LOG_WARNING(Kernel_SVC, "(STUBBED) Assuming process is application"); *result = process->IsApplication(); R_SUCCEED(); diff --git a/src/core/hle/kernel/svc/svc_ipc.cpp b/src/core/hle/kernel/svc/svc_ipc.cpp index 47a3e7bb0..85cc4f561 100644 --- a/src/core/hle/kernel/svc/svc_ipc.cpp +++ b/src/core/hle/kernel/svc/svc_ipc.cpp @@ -48,8 +48,7 @@ Result ReplyAndReceiveImpl(KernelCore& kernel, int32_t* out_index, uintptr_t mes }; // Send the reply. - R_TRY(session->SendReply()); - // R_TRY(session->SendReply(message, buffer_size, message_paddr)); + R_TRY(session->SendReply(message, buffer_size, message_paddr)); } // Receive a message. @@ -85,8 +84,7 @@ Result ReplyAndReceiveImpl(KernelCore& kernel, int32_t* out_index, uintptr_t mes if (R_SUCCEEDED(result)) { KServerSession* session = objs[index]->DynamicCast<KServerSession*>(); if (session != nullptr) { - // result = session->ReceiveRequest(message, buffer_size, message_paddr); - result = session->ReceiveRequest(); + result = session->ReceiveRequest(message, buffer_size, message_paddr); if (ResultNotFound == result) { continue; } diff --git a/src/core/hle/kernel/svc_results.h b/src/core/hle/kernel/svc_results.h index e1ad78607..38e71d516 100644 --- a/src/core/hle/kernel/svc_results.h +++ b/src/core/hle/kernel/svc_results.h @@ -38,7 +38,9 @@ constexpr Result ResultInvalidState{ErrorModule::Kernel, 125}; constexpr Result ResultReservedUsed{ErrorModule::Kernel, 126}; constexpr Result ResultPortClosed{ErrorModule::Kernel, 131}; constexpr Result ResultLimitReached{ErrorModule::Kernel, 132}; +constexpr Result ResultReceiveListBroken{ErrorModule::Kernel, 258}; constexpr Result ResultOutOfAddressSpace{ErrorModule::Kernel, 259}; +constexpr Result ResultMessageTooLarge{ErrorModule::Kernel, 260}; constexpr Result ResultInvalidId{ErrorModule::Kernel, 519}; } // namespace Kernel diff --git a/src/core/hle/service/am/am.cpp b/src/core/hle/service/am/am.cpp index a266d7c21..97eb56ff0 100644 --- a/src/core/hle/service/am/am.cpp +++ b/src/core/hle/service/am/am.cpp @@ -1513,8 +1513,7 @@ void ILibraryAppletCreator::CreateTransferMemoryStorage(HLERequestContext& ctx) return; } - auto transfer_mem = - system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>(handle); + auto transfer_mem = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(handle); if (transfer_mem.IsNull()) { LOG_ERROR(Service_AM, "transfer_mem is a nullptr for handle={:08X}", handle); @@ -1524,8 +1523,7 @@ void ILibraryAppletCreator::CreateTransferMemoryStorage(HLERequestContext& ctx) } std::vector<u8> memory(transfer_mem->GetSize()); - system.ApplicationMemory().ReadBlock(transfer_mem->GetSourceAddress(), memory.data(), - memory.size()); + ctx.GetMemory().ReadBlock(transfer_mem->GetSourceAddress(), memory.data(), memory.size()); IPC::ResponseBuilder rb{ctx, 2, 0, 1}; rb.Push(ResultSuccess); @@ -1547,8 +1545,7 @@ void ILibraryAppletCreator::CreateHandleStorage(HLERequestContext& ctx) { return; } - auto transfer_mem = - system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>(handle); + auto transfer_mem = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(handle); if (transfer_mem.IsNull()) { LOG_ERROR(Service_AM, "transfer_mem is a nullptr for handle={:08X}", handle); @@ -1558,8 +1555,7 @@ void ILibraryAppletCreator::CreateHandleStorage(HLERequestContext& ctx) { } std::vector<u8> memory(transfer_mem->GetSize()); - system.ApplicationMemory().ReadBlock(transfer_mem->GetSourceAddress(), memory.data(), - memory.size()); + ctx.GetMemory().ReadBlock(transfer_mem->GetSourceAddress(), memory.data(), memory.size()); IPC::ResponseBuilder rb{ctx, 2, 0, 1}; rb.Push(ResultSuccess); diff --git a/src/core/hle/service/audio/audren_u.cpp b/src/core/hle/service/audio/audren_u.cpp index 23e56c77a..bd4ca753b 100644 --- a/src/core/hle/service/audio/audren_u.cpp +++ b/src/core/hle/service/audio/audren_u.cpp @@ -454,10 +454,8 @@ void AudRenU::OpenAudioRenderer(HLERequestContext& ctx) { return; } - const auto& handle_table{system.ApplicationProcess()->GetHandleTable()}; - auto process{handle_table.GetObject<Kernel::KProcess>(process_handle)}; - auto transfer_memory{ - process->GetHandleTable().GetObject<Kernel::KTransferMemory>(transfer_memory_handle)}; + auto process{ctx.GetObjectFromHandle<Kernel::KProcess>(process_handle)}; + auto transfer_memory{ctx.GetObjectFromHandle<Kernel::KTransferMemory>(transfer_memory_handle)}; const auto session_id{impl->GetSessionId()}; if (session_id == -1) { diff --git a/src/core/hle/service/audio/hwopus.cpp b/src/core/hle/service/audio/hwopus.cpp index 6a7bf9416..91f33aabd 100644 --- a/src/core/hle/service/audio/hwopus.cpp +++ b/src/core/hle/service/audio/hwopus.cpp @@ -278,9 +278,7 @@ void HwOpus::OpenHardwareOpusDecoder(HLERequestContext& ctx) { auto params = rp.PopRaw<OpusParameters>(); auto transfer_memory_size{rp.Pop<u32>()}; auto transfer_memory_handle{ctx.GetCopyHandle(0)}; - auto transfer_memory{ - system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>( - transfer_memory_handle)}; + auto transfer_memory{ctx.GetObjectFromHandle<Kernel::KTransferMemory>(transfer_memory_handle)}; LOG_DEBUG(Service_Audio, "sample_rate {} channel_count {} transfer_memory_size 0x{:X}", params.sample_rate, params.channel_count, transfer_memory_size); @@ -323,9 +321,7 @@ void HwOpus::OpenHardwareOpusDecoderForMultiStream(HLERequestContext& ctx) { auto transfer_memory_size{rp.Pop<u32>()}; auto transfer_memory_handle{ctx.GetCopyHandle(0)}; - auto transfer_memory{ - system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>( - transfer_memory_handle)}; + auto transfer_memory{ctx.GetObjectFromHandle<Kernel::KTransferMemory>(transfer_memory_handle)}; LOG_DEBUG(Service_Audio, "sample_rate {} channel_count {} total_stream_count {} stereo_stream_count {} " @@ -374,9 +370,7 @@ void HwOpus::OpenHardwareOpusDecoderEx(HLERequestContext& ctx) { auto params = rp.PopRaw<OpusParametersEx>(); auto transfer_memory_size{rp.Pop<u32>()}; auto transfer_memory_handle{ctx.GetCopyHandle(0)}; - auto transfer_memory{ - system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>( - transfer_memory_handle)}; + auto transfer_memory{ctx.GetObjectFromHandle<Kernel::KTransferMemory>(transfer_memory_handle)}; LOG_DEBUG(Service_Audio, "sample_rate {} channel_count {} transfer_memory_size 0x{:X}", params.sample_rate, params.channel_count, transfer_memory_size); @@ -414,9 +408,7 @@ void HwOpus::OpenHardwareOpusDecoderForMultiStreamEx(HLERequestContext& ctx) { auto transfer_memory_size{rp.Pop<u32>()}; auto transfer_memory_handle{ctx.GetCopyHandle(0)}; - auto transfer_memory{ - system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>( - transfer_memory_handle)}; + auto transfer_memory{ctx.GetObjectFromHandle<Kernel::KTransferMemory>(transfer_memory_handle)}; LOG_DEBUG(Service_Audio, "sample_rate {} channel_count {} total_stream_count {} stereo_stream_count {} " diff --git a/src/core/hle/service/fatal/fatal.cpp b/src/core/hle/service/fatal/fatal.cpp index fe2ed8df8..31da86074 100644 --- a/src/core/hle/service/fatal/fatal.cpp +++ b/src/core/hle/service/fatal/fatal.cpp @@ -89,7 +89,7 @@ static void GenerateErrorReport(Core::System& system, Result error_code, const F crash_report += fmt::format(" ESR: {:016x}\n", info.esr); crash_report += fmt::format(" FAR: {:016x}\n", info.far); crash_report += "\nBacktrace:\n"; - for (size_t i = 0; i < info.backtrace_size; i++) { + for (u32 i = 0; i < std::min<u32>(info.backtrace_size, 32); i++) { crash_report += fmt::format(" Backtrace[{:02d}]: {:016x}\n", i, info.backtrace[i]); } diff --git a/src/core/hle/service/hid/hid_server.cpp b/src/core/hle/service/hid/hid_server.cpp index 06a01c02c..3174672af 100644 --- a/src/core/hle/service/hid/hid_server.cpp +++ b/src/core/hle/service/hid/hid_server.cpp @@ -1850,8 +1850,7 @@ void IHidServer::InitializeSevenSixAxisSensor(HLERequestContext& ctx) { ASSERT_MSG(t_mem_1_size == 0x1000, "t_mem_1_size is not 0x1000 bytes"); ASSERT_MSG(t_mem_2_size == 0x7F000, "t_mem_2_size is not 0x7F000 bytes"); - auto t_mem_1 = system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>( - t_mem_1_handle); + auto t_mem_1 = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(t_mem_1_handle); if (t_mem_1.IsNull()) { LOG_ERROR(Service_HID, "t_mem_1 is a nullptr for handle=0x{:08X}", t_mem_1_handle); @@ -1860,8 +1859,7 @@ void IHidServer::InitializeSevenSixAxisSensor(HLERequestContext& ctx) { return; } - auto t_mem_2 = system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>( - t_mem_2_handle); + auto t_mem_2 = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(t_mem_2_handle); if (t_mem_2.IsNull()) { LOG_ERROR(Service_HID, "t_mem_2 is a nullptr for handle=0x{:08X}", t_mem_2_handle); @@ -2142,8 +2140,7 @@ void IHidServer::WritePalmaWaveEntry(HLERequestContext& ctx) { ASSERT_MSG(t_mem_size == 0x3000, "t_mem_size is not 0x3000 bytes"); - auto t_mem = system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>( - t_mem_handle); + auto t_mem = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(t_mem_handle); if (t_mem.IsNull()) { LOG_ERROR(Service_HID, "t_mem is a nullptr for handle=0x{:08X}", t_mem_handle); diff --git a/src/core/hle/service/hid/hidbus.cpp b/src/core/hle/service/hid/hidbus.cpp index 854de2fe2..ffa7e144d 100644 --- a/src/core/hle/service/hid/hidbus.cpp +++ b/src/core/hle/service/hid/hidbus.cpp @@ -448,8 +448,7 @@ void HidBus::EnableJoyPollingReceiveMode(HLERequestContext& ctx) { ASSERT_MSG(t_mem_size == 0x1000, "t_mem_size is not 0x1000 bytes"); - auto t_mem = system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>( - t_mem_handle); + auto t_mem = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(t_mem_handle); if (t_mem.IsNull()) { LOG_ERROR(Service_HID, "t_mem is a nullptr for handle=0x{:08X}", t_mem_handle); diff --git a/src/core/hle/service/hid/irs.cpp b/src/core/hle/service/hid/irs.cpp index 39b9a4474..008debfd1 100644 --- a/src/core/hle/service/hid/irs.cpp +++ b/src/core/hle/service/hid/irs.cpp @@ -197,8 +197,7 @@ void IRS::RunImageTransferProcessor(HLERequestContext& ctx) { const auto parameters{rp.PopRaw<Parameters>()}; const auto t_mem_handle{ctx.GetCopyHandle(0)}; - auto t_mem = system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>( - t_mem_handle); + auto t_mem = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(t_mem_handle); if (t_mem.IsNull()) { LOG_ERROR(Service_IRS, "t_mem is a nullptr for handle=0x{:08X}", t_mem_handle); @@ -444,8 +443,7 @@ void IRS::RunImageTransferExProcessor(HLERequestContext& ctx) { const auto parameters{rp.PopRaw<Parameters>()}; const auto t_mem_handle{ctx.GetCopyHandle(0)}; - auto t_mem = system.ApplicationProcess()->GetHandleTable().GetObject<Kernel::KTransferMemory>( - t_mem_handle); + auto t_mem = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(t_mem_handle); LOG_INFO(Service_IRS, "called, npad_type={}, npad_id={}, transfer_memory_size={}, " diff --git a/src/core/hle/service/hle_ipc.cpp b/src/core/hle/service/hle_ipc.cpp index 38955932c..39df77e43 100644 --- a/src/core/hle/service/hle_ipc.cpp +++ b/src/core/hle/service/hle_ipc.cpp @@ -146,10 +146,7 @@ HLERequestContext::HLERequestContext(Kernel::KernelCore& kernel_, Core::Memory:: HLERequestContext::~HLERequestContext() = default; -void HLERequestContext::ParseCommandBuffer(Kernel::KProcess& process, u32_le* src_cmdbuf, - bool incoming) { - client_handle_table = &process.GetHandleTable(); - +void HLERequestContext::ParseCommandBuffer(u32_le* src_cmdbuf, bool incoming) { IPC::RequestParser rp(src_cmdbuf); command_header = rp.PopRaw<IPC::CommandHeader>(); @@ -162,7 +159,7 @@ void HLERequestContext::ParseCommandBuffer(Kernel::KProcess& process, u32_le* sr if (command_header->enable_handle_descriptor) { handle_descriptor_header = rp.PopRaw<IPC::HandleDescriptorHeader>(); if (handle_descriptor_header->send_current_pid) { - pid = process.GetProcessId(); + pid = thread->GetOwnerProcess()->GetProcessId(); rp.Skip(2, false); } if (incoming) { @@ -270,9 +267,10 @@ void HLERequestContext::ParseCommandBuffer(Kernel::KProcess& process, u32_le* sr rp.Skip(1, false); // The command is actually an u64, but we don't use the high part. } -Result HLERequestContext::PopulateFromIncomingCommandBuffer(Kernel::KProcess& process, - u32_le* src_cmdbuf) { - ParseCommandBuffer(process, src_cmdbuf, true); +Result HLERequestContext::PopulateFromIncomingCommandBuffer(u32_le* src_cmdbuf) { + client_handle_table = &thread->GetOwnerProcess()->GetHandleTable(); + + ParseCommandBuffer(src_cmdbuf, true); if (command_header->IsCloseCommand()) { // Close does not populate the rest of the IPC header @@ -284,9 +282,9 @@ Result HLERequestContext::PopulateFromIncomingCommandBuffer(Kernel::KProcess& pr return ResultSuccess; } -Result HLERequestContext::WriteToOutgoingCommandBuffer(Kernel::KThread& requesting_thread) { +Result HLERequestContext::WriteToOutgoingCommandBuffer() { auto current_offset = handles_offset; - auto& owner_process = *requesting_thread.GetOwnerProcess(); + auto& owner_process = *thread->GetOwnerProcess(); auto& handle_table = owner_process.GetHandleTable(); for (auto& object : outgoing_copy_objects) { @@ -319,7 +317,7 @@ Result HLERequestContext::WriteToOutgoingCommandBuffer(Kernel::KThread& requesti } // Copy the translated command buffer back into the thread's command buffer area. - memory.WriteBlock(requesting_thread.GetTlsAddress(), cmd_buf.data(), write_size * sizeof(u32)); + memory.WriteBlock(thread->GetTlsAddress(), cmd_buf.data(), write_size * sizeof(u32)); return ResultSuccess; } diff --git a/src/core/hle/service/hle_ipc.h b/src/core/hle/service/hle_ipc.h index 18d464c63..40d86943e 100644 --- a/src/core/hle/service/hle_ipc.h +++ b/src/core/hle/service/hle_ipc.h @@ -17,6 +17,7 @@ #include "common/concepts.h" #include "common/swap.h" #include "core/hle/ipc.h" +#include "core/hle/kernel/k_handle_table.h" #include "core/hle/kernel/svc_common.h" union Result; @@ -196,10 +197,10 @@ public: } /// Populates this context with data from the requesting process/thread. - Result PopulateFromIncomingCommandBuffer(Kernel::KProcess& process, u32_le* src_cmdbuf); + Result PopulateFromIncomingCommandBuffer(u32_le* src_cmdbuf); /// Writes data from this context back to the requesting process/thread. - Result WriteToOutgoingCommandBuffer(Kernel::KThread& requesting_thread); + Result WriteToOutgoingCommandBuffer(); [[nodiscard]] u32_le GetHipcCommand() const { return command; @@ -359,8 +360,17 @@ public: return *thread; } - Kernel::KHandleTable& GetClientHandleTable() { - return *client_handle_table; + [[nodiscard]] Core::Memory::Memory& GetMemory() const { + return memory; + } + + template <typename T> + Kernel::KScopedAutoObject<T> GetObjectFromHandle(u32 handle) { + auto obj = client_handle_table->GetObjectForIpc(handle, thread); + if (obj.IsNotNull()) { + return obj->DynamicCast<T*>(); + } + return nullptr; } [[nodiscard]] std::shared_ptr<SessionRequestManager> GetManager() const { @@ -378,7 +388,7 @@ public: private: friend class IPC::ResponseBuilder; - void ParseCommandBuffer(Kernel::KProcess& process, u32_le* src_cmdbuf, bool incoming); + void ParseCommandBuffer(u32_le* src_cmdbuf, bool incoming); std::array<u32, IPC::COMMAND_BUFFER_LENGTH> cmd_buf; Kernel::KServerSession* server_session{}; diff --git a/src/core/hle/service/ipc_helpers.h b/src/core/hle/service/ipc_helpers.h index 0e222362e..4b02872fb 100644 --- a/src/core/hle/service/ipc_helpers.h +++ b/src/core/hle/service/ipc_helpers.h @@ -151,8 +151,8 @@ public: if (manager->IsDomain()) { context->AddDomainObject(std::move(iface)); } else { - kernel.ApplicationProcess()->GetResourceLimit()->Reserve( - Kernel::LimitableResource::SessionCountMax, 1); + ASSERT(Kernel::GetCurrentProcess(kernel).GetResourceLimit()->Reserve( + Kernel::LimitableResource::SessionCountMax, 1)); auto* session = Kernel::KSession::Create(kernel); session->Initialize(nullptr, 0); diff --git a/src/core/hle/service/jit/jit.cpp b/src/core/hle/service/jit/jit.cpp index 65851fc05..77aa6d7d1 100644 --- a/src/core/hle/service/jit/jit.cpp +++ b/src/core/hle/service/jit/jit.cpp @@ -4,11 +4,11 @@ #include "core/arm/debug.h" #include "core/arm/symbols.h" #include "core/core.h" -#include "core/hle/kernel/k_code_memory.h" #include "core/hle/kernel/k_transfer_memory.h" #include "core/hle/result.h" #include "core/hle/service/ipc_helpers.h" #include "core/hle/service/jit/jit.h" +#include "core/hle/service/jit/jit_code_memory.h" #include "core/hle/service/jit/jit_context.h" #include "core/hle/service/server_manager.h" #include "core/hle/service/service.h" @@ -23,9 +23,11 @@ struct CodeRange { class IJitEnvironment final : public ServiceFramework<IJitEnvironment> { public: - explicit IJitEnvironment(Core::System& system_, Kernel::KProcess& process_, CodeRange user_rx, - CodeRange user_ro) - : ServiceFramework{system_, "IJitEnvironment"}, process{&process_}, + explicit IJitEnvironment(Core::System& system_, + Kernel::KScopedAutoObject<Kernel::KProcess>&& process_, + CodeMemory&& user_rx_, CodeMemory&& user_ro_) + : ServiceFramework{system_, "IJitEnvironment"}, process{std::move(process_)}, + user_rx{std::move(user_rx_)}, user_ro{std::move(user_ro_)}, context{system_.ApplicationMemory()} { // clang-format off static const FunctionInfo functions[] = { @@ -39,10 +41,13 @@ public: RegisterHandlers(functions); // Identity map user code range into sysmodule context - configuration.user_ro_memory = user_ro; - configuration.user_rx_memory = user_rx; - configuration.sys_ro_memory = user_ro; - configuration.sys_rx_memory = user_rx; + configuration.user_rx_memory.size = user_rx.GetSize(); + configuration.user_rx_memory.offset = user_rx.GetAddress(); + configuration.user_ro_memory.size = user_ro.GetSize(); + configuration.user_ro_memory.offset = user_ro.GetAddress(); + + configuration.sys_rx_memory = configuration.user_rx_memory; + configuration.sys_ro_memory = configuration.user_ro_memory; } void GenerateCode(HLERequestContext& ctx) { @@ -188,7 +193,7 @@ public: return; } - auto tmem{process->GetHandleTable().GetObject<Kernel::KTransferMemory>(tmem_handle)}; + auto tmem{ctx.GetObjectFromHandle<Kernel::KTransferMemory>(tmem_handle)}; if (tmem.IsNull()) { LOG_ERROR(Service_JIT, "attempted to load plugin with invalid transfer memory handle"); IPC::ResponseBuilder rb{ctx, 2}; @@ -318,6 +323,8 @@ private: } Kernel::KScopedAutoObject<Kernel::KProcess> process; + CodeMemory user_rx; + CodeMemory user_ro; GuestCallbacks callbacks; JITConfiguration configuration; JITContext context; @@ -335,6 +342,7 @@ public: RegisterHandlers(functions); } +private: void CreateJitEnvironment(HLERequestContext& ctx) { LOG_DEBUG(Service_JIT, "called"); @@ -356,11 +364,7 @@ public: return; } - // Fetch using the handle table for the application process here, - // since we are not multiprocess yet. - const auto& handle_table{system.ApplicationProcess()->GetHandleTable()}; - - auto process{handle_table.GetObject<Kernel::KProcess>(process_handle)}; + auto process{ctx.GetObjectFromHandle<Kernel::KProcess>(process_handle)}; if (process.IsNull()) { LOG_ERROR(Service_JIT, "process is null for handle=0x{:08X}", process_handle); IPC::ResponseBuilder rb{ctx, 2}; @@ -368,7 +372,7 @@ public: return; } - auto rx_mem{handle_table.GetObject<Kernel::KCodeMemory>(rx_mem_handle)}; + auto rx_mem{ctx.GetObjectFromHandle<Kernel::KCodeMemory>(rx_mem_handle)}; if (rx_mem.IsNull()) { LOG_ERROR(Service_JIT, "rx_mem is null for handle=0x{:08X}", rx_mem_handle); IPC::ResponseBuilder rb{ctx, 2}; @@ -376,7 +380,7 @@ public: return; } - auto ro_mem{handle_table.GetObject<Kernel::KCodeMemory>(ro_mem_handle)}; + auto ro_mem{ctx.GetObjectFromHandle<Kernel::KCodeMemory>(ro_mem_handle)}; if (ro_mem.IsNull()) { LOG_ERROR(Service_JIT, "ro_mem is null for handle=0x{:08X}", ro_mem_handle); IPC::ResponseBuilder rb{ctx, 2}; @@ -384,20 +388,35 @@ public: return; } - const CodeRange user_rx{ - .offset = GetInteger(rx_mem->GetSourceAddress()), - .size = parameters.rx_size, - }; + CodeMemory rx, ro; + Result res; - const CodeRange user_ro{ - .offset = GetInteger(ro_mem->GetSourceAddress()), - .size = parameters.ro_size, - }; + res = rx.Initialize(*process, *rx_mem, parameters.rx_size, + Kernel::Svc::MemoryPermission::ReadExecute, generate_random); + if (R_FAILED(res)) { + LOG_ERROR(Service_JIT, "rx_mem could not be mapped for handle=0x{:08X}", rx_mem_handle); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(res); + return; + } + + res = ro.Initialize(*process, *ro_mem, parameters.ro_size, + Kernel::Svc::MemoryPermission::Read, generate_random); + if (R_FAILED(res)) { + LOG_ERROR(Service_JIT, "ro_mem could not be mapped for handle=0x{:08X}", ro_mem_handle); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(res); + return; + } IPC::ResponseBuilder rb{ctx, 2, 0, 1}; rb.Push(ResultSuccess); - rb.PushIpcInterface<IJitEnvironment>(system, *process, user_rx, user_ro); + rb.PushIpcInterface<IJitEnvironment>(system, std::move(process), std::move(rx), + std::move(ro)); } + +private: + std::mt19937_64 generate_random{}; }; void LoopProcess(Core::System& system) { diff --git a/src/core/hle/service/jit/jit_code_memory.cpp b/src/core/hle/service/jit/jit_code_memory.cpp new file mode 100644 index 000000000..2b480488a --- /dev/null +++ b/src/core/hle/service/jit/jit_code_memory.cpp @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "core/hle/service/jit/jit_code_memory.h" + +namespace Service::JIT { + +Result CodeMemory::Initialize(Kernel::KProcess& process, Kernel::KCodeMemory& code_memory, + size_t size, Kernel::Svc::MemoryPermission perm, + std::mt19937_64& generate_random) { + auto& page_table = process.GetPageTable(); + const u64 alias_code_start = + GetInteger(page_table.GetAliasCodeRegionStart()) / Kernel::PageSize; + const u64 alias_code_size = page_table.GetAliasCodeRegionSize() / Kernel::PageSize; + + // NOTE: This will retry indefinitely until mapping the code memory succeeds. + while (true) { + // Generate a new trial address. + const u64 mapped_address = + (alias_code_start + (generate_random() % alias_code_size)) * Kernel::PageSize; + + // Try to map the address + R_TRY_CATCH(code_memory.MapToOwner(mapped_address, size, perm)) { + R_CATCH(Kernel::ResultInvalidMemoryRegion) { + // If we could not map here, retry. + continue; + } + } + R_END_TRY_CATCH; + + // Set members. + m_code_memory = std::addressof(code_memory); + m_size = size; + m_address = mapped_address; + m_perm = perm; + + // Open a new reference to the code memory. + m_code_memory->Open(); + + // We succeeded. + R_SUCCEED(); + } +} + +void CodeMemory::Finalize() { + if (m_code_memory) { + R_ASSERT(m_code_memory->UnmapFromOwner(m_address, m_size)); + m_code_memory->Close(); + } + + m_code_memory = nullptr; +} + +} // namespace Service::JIT diff --git a/src/core/hle/service/jit/jit_code_memory.h b/src/core/hle/service/jit/jit_code_memory.h new file mode 100644 index 000000000..6376d4c4e --- /dev/null +++ b/src/core/hle/service/jit/jit_code_memory.h @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include <random> + +#include "core/hle/kernel/k_code_memory.h" + +namespace Service::JIT { + +class CodeMemory { +public: + YUZU_NON_COPYABLE(CodeMemory); + + explicit CodeMemory() = default; + + CodeMemory(CodeMemory&& rhs) { + std::swap(m_code_memory, rhs.m_code_memory); + std::swap(m_size, rhs.m_size); + std::swap(m_address, rhs.m_address); + std::swap(m_perm, rhs.m_perm); + } + + ~CodeMemory() { + this->Finalize(); + } + +public: + Result Initialize(Kernel::KProcess& process, Kernel::KCodeMemory& code_memory, size_t size, + Kernel::Svc::MemoryPermission perm, std::mt19937_64& generate_random); + void Finalize(); + + size_t GetSize() const { + return m_size; + } + + u64 GetAddress() const { + return m_address; + } + +private: + Kernel::KCodeMemory* m_code_memory{}; + size_t m_size{}; + u64 m_address{}; + Kernel::Svc::MemoryPermission m_perm{}; +}; + +} // namespace Service::JIT diff --git a/src/core/hle/service/ro/ro.cpp b/src/core/hle/service/ro/ro.cpp index 17110d3f1..f0658bb5d 100644 --- a/src/core/hle/service/ro/ro.cpp +++ b/src/core/hle/service/ro/ro.cpp @@ -651,10 +651,9 @@ private: void RegisterProcessHandle(HLERequestContext& ctx) { LOG_DEBUG(Service_LDR, "(called)"); - auto process_h = ctx.GetClientHandleTable().GetObject(ctx.GetCopyHandle(0)); + auto process = ctx.GetObjectFromHandle<Kernel::KProcess>(ctx.GetCopyHandle(0)); auto client_pid = ctx.GetPID(); - auto result = interface.RegisterProcessHandle(client_pid, - process_h->DynamicCast<Kernel::KProcess*>()); + auto result = interface.RegisterProcessHandle(client_pid, process.GetPointerUnsafe()); IPC::ResponseBuilder rb{ctx, 2}; rb.Push(result); @@ -671,12 +670,11 @@ private: IPC::RequestParser rp{ctx}; auto params = rp.PopRaw<InputParameters>(); - auto process_h = ctx.GetClientHandleTable().GetObject(ctx.GetCopyHandle(0)); + auto process = ctx.GetObjectFromHandle<Kernel::KProcess>(ctx.GetCopyHandle(0)); auto client_pid = ctx.GetPID(); - auto result = - interface.RegisterProcessModuleInfo(client_pid, params.nrr_address, params.nrr_size, - process_h->DynamicCast<Kernel::KProcess*>()); + auto result = interface.RegisterProcessModuleInfo( + client_pid, params.nrr_address, params.nrr_size, process.GetPointerUnsafe()); IPC::ResponseBuilder rb{ctx, 2}; rb.Push(result); diff --git a/src/core/hle/service/server_manager.cpp b/src/core/hle/service/server_manager.cpp index 6808247a9..15edb23e0 100644 --- a/src/core/hle/service/server_manager.cpp +++ b/src/core/hle/service/server_manager.cpp @@ -47,7 +47,7 @@ ServerManager::~ServerManager() { m_stopped.Wait(); m_threads.clear(); - // Clean up ports. + // Clean up server ports. for (const auto& [port, handler] : m_ports) { port->Close(); } @@ -97,22 +97,15 @@ Result ServerManager::RegisterNamedService(const std::string& service_name, u32 max_sessions) { ASSERT(m_sessions.size() + m_ports.size() < MaximumWaitObjects); - // Add the new server to sm:. - ASSERT(R_SUCCEEDED( - m_system.ServiceManager().RegisterService(service_name, max_sessions, handler_factory))); - - // Get the registered port. - Kernel::KPort* port{}; - ASSERT( - R_SUCCEEDED(m_system.ServiceManager().GetServicePort(std::addressof(port), service_name))); - - // Open a new reference to the server port. - port->GetServerPort().Open(); + // Add the new server to sm: and get the moved server port. + Kernel::KServerPort* server_port{}; + R_ASSERT(m_system.ServiceManager().RegisterService(std::addressof(server_port), service_name, + max_sessions, handler_factory)); // Begin tracking the server port. { std::scoped_lock ll{m_list_mutex}; - m_ports.emplace(std::addressof(port->GetServerPort()), std::move(handler_factory)); + m_ports.emplace(server_port, std::move(handler_factory)); } // Signal the wakeup event. @@ -372,7 +365,7 @@ Result ServerManager::OnSessionEvent(Kernel::KServerSession* session, // Try to receive a message. std::shared_ptr<HLERequestContext> context; - rc = session->ReceiveRequest(&context, manager); + rc = session->ReceiveRequestHLE(&context, manager); // If the session has been closed, we're done. if (rc == Kernel::ResultSessionClosed) { diff --git a/src/core/hle/service/service.cpp b/src/core/hle/service/service.cpp index 00531b021..39124c5fd 100644 --- a/src/core/hle/service/service.cpp +++ b/src/core/hle/service/service.cpp @@ -203,7 +203,7 @@ Result ServiceFrameworkBase::HandleSyncRequest(Kernel::KServerSession& session, // If emulation was shutdown, we are closing service threads, do not write the response back to // memory that may be shutting down as well. if (system.IsPoweredOn()) { - ctx.WriteToOutgoingCommandBuffer(ctx.GetThread()); + ctx.WriteToOutgoingCommandBuffer(); } return result; diff --git a/src/core/hle/service/set/set_sys.cpp b/src/core/hle/service/set/set_sys.cpp index 0653779d5..8e637f963 100644 --- a/src/core/hle/service/set/set_sys.cpp +++ b/src/core/hle/service/set/set_sys.cpp @@ -507,6 +507,14 @@ void SET_SYS::SetTvSettings(HLERequestContext& ctx) { rb.Push(ResultSuccess); } +void SET_SYS::GetDebugModeFlag(HLERequestContext& ctx) { + LOG_DEBUG(Service_SET, "called"); + + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(ResultSuccess); + rb.Push<u32>(0); +} + void SET_SYS::GetQuestFlag(HLERequestContext& ctx) { LOG_WARNING(Service_SET, "(STUBBED) called"); @@ -926,7 +934,7 @@ SET_SYS::SET_SYS(Core::System& system_) : ServiceFramework{system_, "set:sys"}, {59, &SET_SYS::SetNetworkSystemClockContext, "SetNetworkSystemClockContext"}, {60, &SET_SYS::IsUserSystemClockAutomaticCorrectionEnabled, "IsUserSystemClockAutomaticCorrectionEnabled"}, {61, &SET_SYS::SetUserSystemClockAutomaticCorrectionEnabled, "SetUserSystemClockAutomaticCorrectionEnabled"}, - {62, nullptr, "GetDebugModeFlag"}, + {62, &SET_SYS::GetDebugModeFlag, "GetDebugModeFlag"}, {63, &SET_SYS::GetPrimaryAlbumStorage, "GetPrimaryAlbumStorage"}, {64, nullptr, "SetPrimaryAlbumStorage"}, {65, nullptr, "GetUsb30EnableFlag"}, @@ -1143,6 +1151,8 @@ void SET_SYS::StoreSettings() { } void SET_SYS::StoreSettingsThreadFunc(std::stop_token stop_token) { + Common::SetCurrentThreadName("SettingsStore"); + while (Common::StoppableTimedWait(stop_token, std::chrono::minutes(1))) { std::scoped_lock l{m_save_needed_mutex}; if (!std::exchange(m_save_needed, false)) { diff --git a/src/core/hle/service/set/set_sys.h b/src/core/hle/service/set/set_sys.h index 3785d93d8..853f76fce 100644 --- a/src/core/hle/service/set/set_sys.h +++ b/src/core/hle/service/set/set_sys.h @@ -98,6 +98,7 @@ private: void GetSettingsItemValue(HLERequestContext& ctx); void GetTvSettings(HLERequestContext& ctx); void SetTvSettings(HLERequestContext& ctx); + void GetDebugModeFlag(HLERequestContext& ctx); void GetQuestFlag(HLERequestContext& ctx); void GetDeviceTimeZoneLocationName(HLERequestContext& ctx); void SetDeviceTimeZoneLocationName(HLERequestContext& ctx); diff --git a/src/core/hle/service/sm/sm.cpp b/src/core/hle/service/sm/sm.cpp index 296ee6e89..1095dcf6c 100644 --- a/src/core/hle/service/sm/sm.cpp +++ b/src/core/hle/service/sm/sm.cpp @@ -29,8 +29,7 @@ ServiceManager::ServiceManager(Kernel::KernelCore& kernel_) : kernel{kernel_} { ServiceManager::~ServiceManager() { for (auto& [name, port] : service_ports) { - port->GetClientPort().Close(); - port->GetServerPort().Close(); + port->Close(); } if (deferral_event) { @@ -50,8 +49,8 @@ static Result ValidateServiceName(const std::string& name) { return ResultSuccess; } -Result ServiceManager::RegisterService(std::string name, u32 max_sessions, - SessionRequestHandlerFactory handler) { +Result ServiceManager::RegisterService(Kernel::KServerPort** out_server_port, std::string name, + u32 max_sessions, SessionRequestHandlerFactory handler) { R_TRY(ValidateServiceName(name)); std::scoped_lock lk{lock}; @@ -66,13 +65,17 @@ Result ServiceManager::RegisterService(std::string name, u32 max_sessions, // Register the port. Kernel::KPort::Register(kernel, port); - service_ports.emplace(name, port); + service_ports.emplace(name, std::addressof(port->GetClientPort())); registered_services.emplace(name, handler); if (deferral_event) { deferral_event->Signal(); } - return ResultSuccess; + // Set our output. + *out_server_port = std::addressof(port->GetServerPort()); + + // We succeeded. + R_SUCCEED(); } Result ServiceManager::UnregisterService(const std::string& name) { @@ -91,7 +94,8 @@ Result ServiceManager::UnregisterService(const std::string& name) { return ResultSuccess; } -Result ServiceManager::GetServicePort(Kernel::KPort** out_port, const std::string& name) { +Result ServiceManager::GetServicePort(Kernel::KClientPort** out_client_port, + const std::string& name) { R_TRY(ValidateServiceName(name)); std::scoped_lock lk{lock}; @@ -101,7 +105,7 @@ Result ServiceManager::GetServicePort(Kernel::KPort** out_port, const std::strin return Service::SM::ResultNotRegistered; } - *out_port = it->second; + *out_client_port = it->second; return ResultSuccess; } @@ -172,8 +176,8 @@ Result SM::GetServiceImpl(Kernel::KClientSession** out_client_session, HLEReques std::string name(PopServiceName(rp)); // Find the named port. - Kernel::KPort* port{}; - auto port_result = service_manager.GetServicePort(&port, name); + Kernel::KClientPort* client_port{}; + auto port_result = service_manager.GetServicePort(&client_port, name); if (port_result == Service::SM::ResultInvalidServiceName) { LOG_ERROR(Service_SM, "Invalid service name '{}'", name); return Service::SM::ResultInvalidServiceName; @@ -187,7 +191,7 @@ Result SM::GetServiceImpl(Kernel::KClientSession** out_client_session, HLEReques // Create a new session. Kernel::KClientSession* session{}; - if (const auto result = port->GetClientPort().CreateSession(&session); result.IsError()) { + if (const auto result = client_port->CreateSession(&session); result.IsError()) { LOG_ERROR(Service_SM, "called service={} -> error 0x{:08X}", name, result.raw); return result; } @@ -221,7 +225,9 @@ void SM::RegisterServiceImpl(HLERequestContext& ctx, std::string name, u32 max_s LOG_DEBUG(Service_SM, "called with name={}, max_session_count={}, is_light={}", name, max_session_count, is_light); - if (const auto result = service_manager.RegisterService(name, max_session_count, nullptr); + Kernel::KServerPort* server_port{}; + if (const auto result = service_manager.RegisterService(std::addressof(server_port), name, + max_session_count, nullptr); result.IsError()) { LOG_ERROR(Service_SM, "failed to register service with error_code={:08X}", result.raw); IPC::ResponseBuilder rb{ctx, 2}; @@ -229,13 +235,9 @@ void SM::RegisterServiceImpl(HLERequestContext& ctx, std::string name, u32 max_s return; } - auto* port = Kernel::KPort::Create(kernel); - port->Initialize(ServerSessionCountMax, is_light, 0); - SCOPE_EXIT({ port->GetClientPort().Close(); }); - IPC::ResponseBuilder rb{ctx, 2, 0, 1, IPC::ResponseBuilder::Flags::AlwaysMoveHandles}; rb.Push(ResultSuccess); - rb.PushMoveObjects(port->GetServerPort()); + rb.PushMoveObjects(server_port); } void SM::UnregisterService(HLERequestContext& ctx) { diff --git a/src/core/hle/service/sm/sm.h b/src/core/hle/service/sm/sm.h index ff74f588a..4ae32a9c1 100644 --- a/src/core/hle/service/sm/sm.h +++ b/src/core/hle/service/sm/sm.h @@ -56,10 +56,10 @@ public: explicit ServiceManager(Kernel::KernelCore& kernel_); ~ServiceManager(); - Result RegisterService(std::string name, u32 max_sessions, - SessionRequestHandlerFactory handler_factory); + Result RegisterService(Kernel::KServerPort** out_server_port, std::string name, + u32 max_sessions, SessionRequestHandlerFactory handler_factory); Result UnregisterService(const std::string& name); - Result GetServicePort(Kernel::KPort** out_port, const std::string& name); + Result GetServicePort(Kernel::KClientPort** out_client_port, const std::string& name); template <Common::DerivedFrom<SessionRequestHandler> T> std::shared_ptr<T> GetService(const std::string& service_name) const { @@ -84,7 +84,7 @@ private: /// Map of registered services, retrieved using GetServicePort. std::mutex lock; std::unordered_map<std::string, SessionRequestHandlerFactory> registered_services; - std::unordered_map<std::string, Kernel::KPort*> service_ports; + std::unordered_map<std::string, Kernel::KClientPort*> service_ports; /// Kernel context Kernel::KernelCore& kernel; diff --git a/src/core/hle/service/sm/sm_controller.cpp b/src/core/hle/service/sm/sm_controller.cpp index 7dce28fe0..7f0fb91d0 100644 --- a/src/core/hle/service/sm/sm_controller.cpp +++ b/src/core/hle/service/sm/sm_controller.cpp @@ -28,7 +28,6 @@ void Controller::ConvertCurrentObjectToDomain(HLERequestContext& ctx) { void Controller::CloneCurrentObject(HLERequestContext& ctx) { LOG_DEBUG(Service, "called"); - auto& process = *ctx.GetThread().GetOwnerProcess(); auto session_manager = ctx.GetManager(); // FIXME: this is duplicated from the SVC, it should just call it instead @@ -36,11 +35,11 @@ void Controller::CloneCurrentObject(HLERequestContext& ctx) { // Reserve a new session from the process resource limit. Kernel::KScopedResourceReservation session_reservation( - &process, Kernel::LimitableResource::SessionCountMax); + Kernel::GetCurrentProcessPointer(kernel), Kernel::LimitableResource::SessionCountMax); ASSERT(session_reservation.Succeeded()); // Create the session. - Kernel::KSession* session = Kernel::KSession::Create(system.Kernel()); + Kernel::KSession* session = Kernel::KSession::Create(kernel); ASSERT(session != nullptr); // Initialize the session. @@ -50,7 +49,7 @@ void Controller::CloneCurrentObject(HLERequestContext& ctx) { session_reservation.Commit(); // Register the session. - Kernel::KSession::Register(system.Kernel(), session); + Kernel::KSession::Register(kernel, session); // Register with server manager. session_manager->GetServerManager().RegisterSession(&session->GetServerSession(), diff --git a/src/core/loader/deconstructed_rom_directory.cpp b/src/core/loader/deconstructed_rom_directory.cpp index 60ee78e89..c9f8707b7 100644 --- a/src/core/loader/deconstructed_rom_directory.cpp +++ b/src/core/loader/deconstructed_rom_directory.cpp @@ -129,9 +129,10 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect } metadata.Print(); - // Enable NCE only for programs with 39-bit address space. + // Enable NCE only for applications with 39-bit address space. const bool is_39bit = metadata.GetAddressSpaceType() == FileSys::ProgramAddressSpaceType::Is39Bit; + const bool is_application = metadata.GetPoolPartition() == FileSys::PoolPartition::Application; Settings::SetNceEnabled(is_39bit); const std::array static_modules = {"rtld", "main", "subsdk0", "subsdk1", "subsdk2", @@ -147,7 +148,7 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect const auto GetPatcher = [&](size_t i) -> Core::NCE::Patcher* { #ifdef HAS_NCE - if (Settings::IsNceEnabled()) { + if (is_application && Settings::IsNceEnabled()) { return &module_patchers[i]; } #endif @@ -175,7 +176,7 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect // Enable direct memory mapping in case of NCE. const u64 fastmem_base = [&]() -> size_t { - if (Settings::IsNceEnabled()) { + if (is_application && Settings::IsNceEnabled()) { auto& buffer = system.DeviceMemory().buffer; buffer.EnableDirectMappedAddress(); return reinterpret_cast<u64>(buffer.VirtualBasePointer()); diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 169bf4c8c..8176a41be 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -10,6 +10,7 @@ #include "common/assert.h" #include "common/atomic_ops.h" #include "common/common_types.h" +#include "common/heap_tracker.h" #include "common/logging/log.h" #include "common/page_table.h" #include "common/scope_exit.h" @@ -45,11 +46,25 @@ struct Memory::Impl { void SetCurrentPageTable(Kernel::KProcess& process) { current_page_table = &process.GetPageTable().GetImpl(); - current_page_table->fastmem_arena = system.DeviceMemory().buffer.VirtualBasePointer(); + + if (std::addressof(process) == system.ApplicationProcess() && + Settings::IsFastmemEnabled()) { + current_page_table->fastmem_arena = system.DeviceMemory().buffer.VirtualBasePointer(); + } else { + current_page_table->fastmem_arena = nullptr; + } + +#ifdef __linux__ + heap_tracker.emplace(system.DeviceMemory().buffer); + buffer = std::addressof(*heap_tracker); +#else + buffer = std::addressof(system.DeviceMemory().buffer); +#endif } void MapMemoryRegion(Common::PageTable& page_table, Common::ProcessAddress base, u64 size, - Common::PhysicalAddress target, Common::MemoryPermission perms) { + Common::PhysicalAddress target, Common::MemoryPermission perms, + bool separate_heap) { ASSERT_MSG((size & YUZU_PAGEMASK) == 0, "non-page aligned size: {:016X}", size); ASSERT_MSG((base & YUZU_PAGEMASK) == 0, "non-page aligned base: {:016X}", GetInteger(base)); ASSERT_MSG(target >= DramMemoryMap::Base, "Out of bounds target: {:016X}", @@ -57,20 +72,21 @@ struct Memory::Impl { MapPages(page_table, base / YUZU_PAGESIZE, size / YUZU_PAGESIZE, target, Common::PageType::Memory); - if (Settings::IsFastmemEnabled()) { - system.DeviceMemory().buffer.Map(GetInteger(base), - GetInteger(target) - DramMemoryMap::Base, size, perms); + if (current_page_table->fastmem_arena) { + buffer->Map(GetInteger(base), GetInteger(target) - DramMemoryMap::Base, size, perms, + separate_heap); } } - void UnmapRegion(Common::PageTable& page_table, Common::ProcessAddress base, u64 size) { + void UnmapRegion(Common::PageTable& page_table, Common::ProcessAddress base, u64 size, + bool separate_heap) { ASSERT_MSG((size & YUZU_PAGEMASK) == 0, "non-page aligned size: {:016X}", size); ASSERT_MSG((base & YUZU_PAGEMASK) == 0, "non-page aligned base: {:016X}", GetInteger(base)); MapPages(page_table, base / YUZU_PAGESIZE, size / YUZU_PAGESIZE, 0, Common::PageType::Unmapped); - if (Settings::IsFastmemEnabled()) { - system.DeviceMemory().buffer.Unmap(GetInteger(base), size); + if (current_page_table->fastmem_arena) { + buffer->Unmap(GetInteger(base), size, separate_heap); } } @@ -79,17 +95,7 @@ struct Memory::Impl { ASSERT_MSG((size & YUZU_PAGEMASK) == 0, "non-page aligned size: {:016X}", size); ASSERT_MSG((vaddr & YUZU_PAGEMASK) == 0, "non-page aligned base: {:016X}", vaddr); - if (!Settings::IsFastmemEnabled()) { - return; - } - - const bool is_r = True(perms & Common::MemoryPermission::Read); - const bool is_w = True(perms & Common::MemoryPermission::Write); - const bool is_x = - True(perms & Common::MemoryPermission::Execute) && Settings::IsNceEnabled(); - - if (!current_page_table) { - system.DeviceMemory().buffer.Protect(vaddr, size, is_r, is_w, is_x); + if (!current_page_table->fastmem_arena) { return; } @@ -101,8 +107,7 @@ struct Memory::Impl { switch (page_type) { case Common::PageType::RasterizerCachedMemory: if (protect_bytes > 0) { - system.DeviceMemory().buffer.Protect(protect_begin, protect_bytes, is_r, is_w, - is_x); + buffer->Protect(protect_begin, protect_bytes, perms); protect_bytes = 0; } break; @@ -115,7 +120,7 @@ struct Memory::Impl { } if (protect_bytes > 0) { - system.DeviceMemory().buffer.Protect(protect_begin, protect_bytes, is_r, is_w, is_x); + buffer->Protect(protect_begin, protect_bytes, perms); } } @@ -239,7 +244,7 @@ struct Memory::Impl { bool WalkBlock(const Common::ProcessAddress addr, const std::size_t size, auto on_unmapped, auto on_memory, auto on_rasterizer, auto increment) { - const auto& page_table = system.ApplicationProcess()->GetPageTable().GetImpl(); + const auto& page_table = *current_page_table; std::size_t remaining_size = size; std::size_t page_index = addr >> YUZU_PAGEBITS; std::size_t page_offset = addr & YUZU_PAGEMASK; @@ -484,8 +489,10 @@ struct Memory::Impl { return; } - if (Settings::IsFastmemEnabled()) { - system.DeviceMemory().buffer.Protect(vaddr, size, !debug, !debug); + if (current_page_table->fastmem_arena) { + const auto perm{debug ? Common::MemoryPermission{} + : Common::MemoryPermission::ReadWrite}; + buffer->Protect(vaddr, size, perm); } // Iterate over a contiguous CPU address space, marking/unmarking the region. @@ -541,10 +548,15 @@ struct Memory::Impl { return; } - if (Settings::IsFastmemEnabled()) { - const bool is_read_enable = - !Settings::values.use_reactive_flushing.GetValue() || !cached; - system.DeviceMemory().buffer.Protect(vaddr, size, is_read_enable, !cached); + if (current_page_table->fastmem_arena) { + Common::MemoryPermission perm{}; + if (!Settings::values.use_reactive_flushing.GetValue() || !cached) { + perm |= Common::MemoryPermission::Read; + } + if (!cached) { + perm |= Common::MemoryPermission::Write; + } + buffer->Protect(vaddr, size, perm); } // Iterate over a contiguous CPU address space, which corresponds to the specified GPU @@ -855,6 +867,13 @@ struct Memory::Impl { std::array<GPUDirtyState, Core::Hardware::NUM_CPU_CORES> rasterizer_write_areas{}; std::span<Core::GPUDirtyMemoryManager> gpu_dirty_managers; std::mutex sys_core_guard; + + std::optional<Common::HeapTracker> heap_tracker; +#ifdef __linux__ + Common::HeapTracker* buffer{}; +#else + Common::HostMemory* buffer{}; +#endif }; Memory::Memory(Core::System& system_) : system{system_} { @@ -872,12 +891,14 @@ void Memory::SetCurrentPageTable(Kernel::KProcess& process) { } void Memory::MapMemoryRegion(Common::PageTable& page_table, Common::ProcessAddress base, u64 size, - Common::PhysicalAddress target, Common::MemoryPermission perms) { - impl->MapMemoryRegion(page_table, base, size, target, perms); + Common::PhysicalAddress target, Common::MemoryPermission perms, + bool separate_heap) { + impl->MapMemoryRegion(page_table, base, size, target, perms, separate_heap); } -void Memory::UnmapRegion(Common::PageTable& page_table, Common::ProcessAddress base, u64 size) { - impl->UnmapRegion(page_table, base, size); +void Memory::UnmapRegion(Common::PageTable& page_table, Common::ProcessAddress base, u64 size, + bool separate_heap) { + impl->UnmapRegion(page_table, base, size, separate_heap); } void Memory::ProtectRegion(Common::PageTable& page_table, Common::ProcessAddress vaddr, u64 size, @@ -886,8 +907,7 @@ void Memory::ProtectRegion(Common::PageTable& page_table, Common::ProcessAddress } bool Memory::IsValidVirtualAddress(const Common::ProcessAddress vaddr) const { - const Kernel::KProcess& process = *system.ApplicationProcess(); - const auto& page_table = process.GetPageTable().GetImpl(); + const auto& page_table = *impl->current_page_table; const size_t page = vaddr >> YUZU_PAGEBITS; if (page >= page_table.pointers.size()) { return false; @@ -1048,7 +1068,9 @@ void Memory::FlushRegion(Common::ProcessAddress dest_addr, size_t size) { } bool Memory::InvalidateNCE(Common::ProcessAddress vaddr, size_t size) { - bool mapped = true; + [[maybe_unused]] bool mapped = true; + [[maybe_unused]] bool rasterizer = false; + u8* const ptr = impl->GetPointerImpl( GetInteger(vaddr), [&] { @@ -1056,8 +1078,26 @@ bool Memory::InvalidateNCE(Common::ProcessAddress vaddr, size_t size) { GetInteger(vaddr)); mapped = false; }, - [&] { impl->system.GPU().InvalidateRegion(GetInteger(vaddr), size); }); + [&] { + impl->system.GPU().InvalidateRegion(GetInteger(vaddr), size); + rasterizer = true; + }); + +#ifdef __linux__ + if (!rasterizer && mapped) { + impl->buffer->DeferredMapSeparateHeap(GetInteger(vaddr)); + } +#endif + return mapped && ptr != nullptr; } +bool Memory::InvalidateSeparateHeap(void* fault_address) { +#ifdef __linux__ + return impl->buffer->DeferredMapSeparateHeap(static_cast<u8*>(fault_address)); +#else + return false; +#endif +} + } // namespace Core::Memory diff --git a/src/core/memory.h b/src/core/memory.h index c1879e78f..3e4d03f57 100644 --- a/src/core/memory.h +++ b/src/core/memory.h @@ -86,7 +86,8 @@ public: * @param perms The permissions to map the memory with. */ void MapMemoryRegion(Common::PageTable& page_table, Common::ProcessAddress base, u64 size, - Common::PhysicalAddress target, Common::MemoryPermission perms); + Common::PhysicalAddress target, Common::MemoryPermission perms, + bool separate_heap); /** * Unmaps a region of the emulated process address space. @@ -95,7 +96,8 @@ public: * @param base The address to begin unmapping at. * @param size The amount of bytes to unmap. */ - void UnmapRegion(Common::PageTable& page_table, Common::ProcessAddress base, u64 size); + void UnmapRegion(Common::PageTable& page_table, Common::ProcessAddress base, u64 size, + bool separate_heap); /** * Protects a region of the emulated process address space with the new permissions. @@ -486,6 +488,7 @@ public: void SetGPUDirtyManagers(std::span<Core::GPUDirtyMemoryManager> managers); void InvalidateRegion(Common::ProcessAddress dest_addr, size_t size); bool InvalidateNCE(Common::ProcessAddress vaddr, size_t size); + bool InvalidateSeparateHeap(void* fault_address); void FlushRegion(Common::ProcessAddress dest_addr, size_t size); private: diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp index ed023fcfe..89ebab08e 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp @@ -96,9 +96,9 @@ Id ImageType(EmitContext& ctx, const ImageDescriptor& desc, Id sampled_type) { } Id DefineVariable(EmitContext& ctx, Id type, std::optional<spv::BuiltIn> builtin, - spv::StorageClass storage_class) { + spv::StorageClass storage_class, std::optional<Id> initializer = std::nullopt) { const Id pointer_type{ctx.TypePointer(storage_class, type)}; - const Id id{ctx.AddGlobalVariable(pointer_type, storage_class)}; + const Id id{ctx.AddGlobalVariable(pointer_type, storage_class, initializer)}; if (builtin) { ctx.Decorate(id, spv::Decoration::BuiltIn, *builtin); } @@ -144,11 +144,12 @@ Id DefineInput(EmitContext& ctx, Id type, bool per_invocation, } Id DefineOutput(EmitContext& ctx, Id type, std::optional<u32> invocations, - std::optional<spv::BuiltIn> builtin = std::nullopt) { + std::optional<spv::BuiltIn> builtin = std::nullopt, + std::optional<Id> initializer = std::nullopt) { if (invocations && ctx.stage == Stage::TessellationControl) { type = ctx.TypeArray(type, ctx.Const(*invocations)); } - return DefineVariable(ctx, type, builtin, spv::StorageClass::Output); + return DefineVariable(ctx, type, builtin, spv::StorageClass::Output, initializer); } void DefineGenericOutput(EmitContext& ctx, size_t index, std::optional<u32> invocations) { @@ -811,10 +812,14 @@ void EmitContext::DefineAttributeMemAccess(const Info& info) { labels.push_back(OpLabel()); } if (info.stores.ClipDistances()) { - literals.push_back(static_cast<u32>(IR::Attribute::ClipDistance0) >> 2); - labels.push_back(OpLabel()); - literals.push_back(static_cast<u32>(IR::Attribute::ClipDistance4) >> 2); - labels.push_back(OpLabel()); + if (profile.max_user_clip_distances >= 4) { + literals.push_back(static_cast<u32>(IR::Attribute::ClipDistance0) >> 2); + labels.push_back(OpLabel()); + } + if (profile.max_user_clip_distances >= 8) { + literals.push_back(static_cast<u32>(IR::Attribute::ClipDistance4) >> 2); + labels.push_back(OpLabel()); + } } OpSelectionMerge(end_block, spv::SelectionControlMask::MaskNone); OpSwitch(compare_index, default_label, literals, labels); @@ -843,17 +848,21 @@ void EmitContext::DefineAttributeMemAccess(const Info& info) { ++label_index; } if (info.stores.ClipDistances()) { - AddLabel(labels[label_index]); - const Id pointer{OpAccessChain(output_f32, clip_distances, masked_index)}; - OpStore(pointer, store_value); - OpReturn(); - ++label_index; - AddLabel(labels[label_index]); - const Id fixed_index{OpIAdd(U32[1], masked_index, Const(4U))}; - const Id pointer2{OpAccessChain(output_f32, clip_distances, fixed_index)}; - OpStore(pointer2, store_value); - OpReturn(); - ++label_index; + if (profile.max_user_clip_distances >= 4) { + AddLabel(labels[label_index]); + const Id pointer{OpAccessChain(output_f32, clip_distances, masked_index)}; + OpStore(pointer, store_value); + OpReturn(); + ++label_index; + } + if (profile.max_user_clip_distances >= 8) { + AddLabel(labels[label_index]); + const Id fixed_index{OpIAdd(U32[1], masked_index, Const(4U))}; + const Id pointer{OpAccessChain(output_f32, clip_distances, fixed_index)}; + OpStore(pointer, store_value); + OpReturn(); + ++label_index; + } } AddLabel(end_block); OpUnreachable(); @@ -1532,9 +1541,16 @@ void EmitContext::DefineOutputs(const IR::Program& program) { if (stage == Stage::Fragment) { throw NotImplementedException("Storing ClipDistance in fragment stage"); } - const Id type{TypeArray( - F32[1], Const(std::min(info.used_clip_distances, profile.max_user_clip_distances)))}; - clip_distances = DefineOutput(*this, type, invocations, spv::BuiltIn::ClipDistance); + if (profile.max_user_clip_distances > 0) { + const u32 used{std::min(profile.max_user_clip_distances, 8u)}; + const std::array<Id, 8> zero{f32_zero_value, f32_zero_value, f32_zero_value, + f32_zero_value, f32_zero_value, f32_zero_value, + f32_zero_value, f32_zero_value}; + const Id type{TypeArray(F32[1], Const(used))}; + const Id initializer{ConstantComposite(type, std::span(zero).subspan(0, used))}; + clip_distances = + DefineOutput(*this, type, invocations, spv::BuiltIn::ClipDistance, initializer); + } } if (info.stores[IR::Attribute::Layer] && (profile.support_viewport_index_layer_non_geometry || stage == Stage::Geometry)) { diff --git a/src/tests/common/host_memory.cpp b/src/tests/common/host_memory.cpp index 1a28e862b..cb040c942 100644 --- a/src/tests/common/host_memory.cpp +++ b/src/tests/common/host_memory.cpp @@ -12,6 +12,7 @@ using namespace Common::Literals; static constexpr size_t VIRTUAL_SIZE = 1ULL << 39; static constexpr size_t BACKING_SIZE = 4_GiB; static constexpr auto PERMS = Common::MemoryPermission::ReadWrite; +static constexpr auto HEAP = false; TEST_CASE("HostMemory: Initialize and deinitialize", "[common]") { { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); } @@ -20,7 +21,7 @@ TEST_CASE("HostMemory: Initialize and deinitialize", "[common]") { TEST_CASE("HostMemory: Simple map", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x5000, 0x8000, 0x1000, PERMS); + mem.Map(0x5000, 0x8000, 0x1000, PERMS, HEAP); volatile u8* const data = mem.VirtualBasePointer() + 0x5000; data[0] = 50; @@ -29,8 +30,8 @@ TEST_CASE("HostMemory: Simple map", "[common]") { TEST_CASE("HostMemory: Simple mirror map", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x5000, 0x3000, 0x2000, PERMS); - mem.Map(0x8000, 0x4000, 0x1000, PERMS); + mem.Map(0x5000, 0x3000, 0x2000, PERMS, HEAP); + mem.Map(0x8000, 0x4000, 0x1000, PERMS, HEAP); volatile u8* const mirror_a = mem.VirtualBasePointer() + 0x5000; volatile u8* const mirror_b = mem.VirtualBasePointer() + 0x8000; @@ -40,116 +41,116 @@ TEST_CASE("HostMemory: Simple mirror map", "[common]") { TEST_CASE("HostMemory: Simple unmap", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x5000, 0x3000, 0x2000, PERMS); + mem.Map(0x5000, 0x3000, 0x2000, PERMS, HEAP); volatile u8* const data = mem.VirtualBasePointer() + 0x5000; data[75] = 50; REQUIRE(data[75] == 50); - mem.Unmap(0x5000, 0x2000); + mem.Unmap(0x5000, 0x2000, HEAP); } TEST_CASE("HostMemory: Simple unmap and remap", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x5000, 0x3000, 0x2000, PERMS); + mem.Map(0x5000, 0x3000, 0x2000, PERMS, HEAP); volatile u8* const data = mem.VirtualBasePointer() + 0x5000; data[0] = 50; REQUIRE(data[0] == 50); - mem.Unmap(0x5000, 0x2000); + mem.Unmap(0x5000, 0x2000, HEAP); - mem.Map(0x5000, 0x3000, 0x2000, PERMS); + mem.Map(0x5000, 0x3000, 0x2000, PERMS, HEAP); REQUIRE(data[0] == 50); - mem.Map(0x7000, 0x2000, 0x5000, PERMS); + mem.Map(0x7000, 0x2000, 0x5000, PERMS, HEAP); REQUIRE(data[0x3000] == 50); } TEST_CASE("HostMemory: Nieche allocation", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x0000, 0, 0x20000, PERMS); - mem.Unmap(0x0000, 0x4000); - mem.Map(0x1000, 0, 0x2000, PERMS); - mem.Map(0x3000, 0, 0x1000, PERMS); - mem.Map(0, 0, 0x1000, PERMS); + mem.Map(0x0000, 0, 0x20000, PERMS, HEAP); + mem.Unmap(0x0000, 0x4000, HEAP); + mem.Map(0x1000, 0, 0x2000, PERMS, HEAP); + mem.Map(0x3000, 0, 0x1000, PERMS, HEAP); + mem.Map(0, 0, 0x1000, PERMS, HEAP); } TEST_CASE("HostMemory: Full unmap", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x8000, 0, 0x4000, PERMS); - mem.Unmap(0x8000, 0x4000); - mem.Map(0x6000, 0, 0x16000, PERMS); + mem.Map(0x8000, 0, 0x4000, PERMS, HEAP); + mem.Unmap(0x8000, 0x4000, HEAP); + mem.Map(0x6000, 0, 0x16000, PERMS, HEAP); } TEST_CASE("HostMemory: Right out of bounds unmap", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x0000, 0, 0x4000, PERMS); - mem.Unmap(0x2000, 0x4000); - mem.Map(0x2000, 0x80000, 0x4000, PERMS); + mem.Map(0x0000, 0, 0x4000, PERMS, HEAP); + mem.Unmap(0x2000, 0x4000, HEAP); + mem.Map(0x2000, 0x80000, 0x4000, PERMS, HEAP); } TEST_CASE("HostMemory: Left out of bounds unmap", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x8000, 0, 0x4000, PERMS); - mem.Unmap(0x6000, 0x4000); - mem.Map(0x8000, 0, 0x2000, PERMS); + mem.Map(0x8000, 0, 0x4000, PERMS, HEAP); + mem.Unmap(0x6000, 0x4000, HEAP); + mem.Map(0x8000, 0, 0x2000, PERMS, HEAP); } TEST_CASE("HostMemory: Multiple placeholder unmap", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x0000, 0, 0x4000, PERMS); - mem.Map(0x4000, 0, 0x1b000, PERMS); - mem.Unmap(0x3000, 0x1c000); - mem.Map(0x3000, 0, 0x20000, PERMS); + mem.Map(0x0000, 0, 0x4000, PERMS, HEAP); + mem.Map(0x4000, 0, 0x1b000, PERMS, HEAP); + mem.Unmap(0x3000, 0x1c000, HEAP); + mem.Map(0x3000, 0, 0x20000, PERMS, HEAP); } TEST_CASE("HostMemory: Unmap between placeholders", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x0000, 0, 0x4000, PERMS); - mem.Map(0x4000, 0, 0x4000, PERMS); - mem.Unmap(0x2000, 0x4000); - mem.Map(0x2000, 0, 0x4000, PERMS); + mem.Map(0x0000, 0, 0x4000, PERMS, HEAP); + mem.Map(0x4000, 0, 0x4000, PERMS, HEAP); + mem.Unmap(0x2000, 0x4000, HEAP); + mem.Map(0x2000, 0, 0x4000, PERMS, HEAP); } TEST_CASE("HostMemory: Unmap to origin", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x4000, 0, 0x4000, PERMS); - mem.Map(0x8000, 0, 0x4000, PERMS); - mem.Unmap(0x4000, 0x4000); - mem.Map(0, 0, 0x4000, PERMS); - mem.Map(0x4000, 0, 0x4000, PERMS); + mem.Map(0x4000, 0, 0x4000, PERMS, HEAP); + mem.Map(0x8000, 0, 0x4000, PERMS, HEAP); + mem.Unmap(0x4000, 0x4000, HEAP); + mem.Map(0, 0, 0x4000, PERMS, HEAP); + mem.Map(0x4000, 0, 0x4000, PERMS, HEAP); } TEST_CASE("HostMemory: Unmap to right", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x4000, 0, 0x4000, PERMS); - mem.Map(0x8000, 0, 0x4000, PERMS); - mem.Unmap(0x8000, 0x4000); - mem.Map(0x8000, 0, 0x4000, PERMS); + mem.Map(0x4000, 0, 0x4000, PERMS, HEAP); + mem.Map(0x8000, 0, 0x4000, PERMS, HEAP); + mem.Unmap(0x8000, 0x4000, HEAP); + mem.Map(0x8000, 0, 0x4000, PERMS, HEAP); } TEST_CASE("HostMemory: Partial right unmap check bindings", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x4000, 0x10000, 0x4000, PERMS); + mem.Map(0x4000, 0x10000, 0x4000, PERMS, HEAP); volatile u8* const ptr = mem.VirtualBasePointer() + 0x4000; ptr[0x1000] = 17; - mem.Unmap(0x6000, 0x2000); + mem.Unmap(0x6000, 0x2000, HEAP); REQUIRE(ptr[0x1000] == 17); } TEST_CASE("HostMemory: Partial left unmap check bindings", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x4000, 0x10000, 0x4000, PERMS); + mem.Map(0x4000, 0x10000, 0x4000, PERMS, HEAP); volatile u8* const ptr = mem.VirtualBasePointer() + 0x4000; ptr[0x3000] = 19; ptr[0x3fff] = 12; - mem.Unmap(0x4000, 0x2000); + mem.Unmap(0x4000, 0x2000, HEAP); REQUIRE(ptr[0x3000] == 19); REQUIRE(ptr[0x3fff] == 12); @@ -157,13 +158,13 @@ TEST_CASE("HostMemory: Partial left unmap check bindings", "[common]") { TEST_CASE("HostMemory: Partial middle unmap check bindings", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x4000, 0x10000, 0x4000, PERMS); + mem.Map(0x4000, 0x10000, 0x4000, PERMS, HEAP); volatile u8* const ptr = mem.VirtualBasePointer() + 0x4000; ptr[0x0000] = 19; ptr[0x3fff] = 12; - mem.Unmap(0x1000, 0x2000); + mem.Unmap(0x1000, 0x2000, HEAP); REQUIRE(ptr[0x0000] == 19); REQUIRE(ptr[0x3fff] == 12); @@ -171,14 +172,14 @@ TEST_CASE("HostMemory: Partial middle unmap check bindings", "[common]") { TEST_CASE("HostMemory: Partial sparse middle unmap and check bindings", "[common]") { HostMemory mem(BACKING_SIZE, VIRTUAL_SIZE); - mem.Map(0x4000, 0x10000, 0x2000, PERMS); - mem.Map(0x6000, 0x20000, 0x2000, PERMS); + mem.Map(0x4000, 0x10000, 0x2000, PERMS, HEAP); + mem.Map(0x6000, 0x20000, 0x2000, PERMS, HEAP); volatile u8* const ptr = mem.VirtualBasePointer() + 0x4000; ptr[0x0000] = 19; ptr[0x3fff] = 12; - mem.Unmap(0x5000, 0x2000); + mem.Unmap(0x5000, 0x2000, HEAP); REQUIRE(ptr[0x0000] == 19); REQUIRE(ptr[0x3fff] == 12); diff --git a/src/video_core/macro/macro_hle.cpp b/src/video_core/macro/macro_hle.cpp index 046c8085e..46e853e04 100644 --- a/src/video_core/macro/macro_hle.cpp +++ b/src/video_core/macro/macro_hle.cpp @@ -327,12 +327,13 @@ public: explicit HLE_DrawIndirectByteCount(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {} void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override { + const bool force = maxwell3d.Rasterizer().HasDrawTransformFeedback(); + auto topology = static_cast<Maxwell3D::Regs::PrimitiveTopology>(parameters[0] & 0xFFFFU); - if (!maxwell3d.AnyParametersDirty() || !IsTopologySafe(topology)) { + if (!force && (!maxwell3d.AnyParametersDirty() || !IsTopologySafe(topology))) { Fallback(parameters); return; } - auto& params = maxwell3d.draw_manager->GetIndirectParams(); params.is_byte_count = true; params.is_indexed = false; @@ -503,6 +504,8 @@ public: maxwell3d.CallMethod(static_cast<size_t>(MAXWELL3D_REG_INDEX(launch_dma)), 0x1011, true); maxwell3d.CallMethod(static_cast<size_t>(MAXWELL3D_REG_INDEX(inline_data)), regs.transform_feedback.controls[0].stride, true); + + maxwell3d.Rasterizer().RegisterTransformFeedback(regs.upload.dest.Address()); } }; diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index af1469147..49224ca85 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -173,5 +173,13 @@ public: virtual void BindChannel(Tegra::Control::ChannelState& channel) {} virtual void ReleaseChannel(s32 channel_id) {} + + /// Register the address as a Transform Feedback Object + virtual void RegisterTransformFeedback(GPUVAddr tfb_object_addr) {} + + /// Returns true when the rasterizer has Draw Transform Feedback capabilities + virtual bool HasDrawTransformFeedback() { + return false; + } }; } // namespace VideoCore diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.cpp b/src/video_core/renderer_opengl/gl_buffer_cache.cpp index b787b6994..517ac14dd 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.cpp +++ b/src/video_core/renderer_opengl/gl_buffer_cache.cpp @@ -376,4 +376,15 @@ void BufferCacheRuntime::BindImageBuffer(Buffer& buffer, u32 offset, u32 size, P *image_handles++ = buffer.View(offset, size, format); } +void BufferCacheRuntime::BindTransformFeedbackObject(GPUVAddr tfb_object_addr) { + OGLTransformFeedback& tfb_object = tfb_objects[tfb_object_addr]; + tfb_object.Create(); + glBindTransformFeedback(GL_TRANSFORM_FEEDBACK, tfb_object.handle); +} + +GLuint BufferCacheRuntime::GetTransformFeedbackObject(GPUVAddr tfb_object_addr) { + ASSERT(tfb_objects.contains(tfb_object_addr)); + return tfb_objects[tfb_object_addr].handle; +} + } // namespace OpenGL diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.h b/src/video_core/renderer_opengl/gl_buffer_cache.h index 1e8708f59..2c18de166 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.h +++ b/src/video_core/renderer_opengl/gl_buffer_cache.h @@ -5,6 +5,7 @@ #include <array> #include <span> +#include <unordered_map> #include "common/common_types.h" #include "video_core/buffer_cache/buffer_cache_base.h" @@ -121,6 +122,9 @@ public: void BindImageBuffer(Buffer& buffer, u32 offset, u32 size, VideoCore::Surface::PixelFormat format); + void BindTransformFeedbackObject(GPUVAddr tfb_object_addr); + GLuint GetTransformFeedbackObject(GPUVAddr tfb_object_addr); + u64 GetDeviceMemoryUsage() const; void BindFastUniformBuffer(size_t stage, u32 binding_index, u32 size) { @@ -233,6 +237,7 @@ private: u32 index_buffer_offset = 0; u64 device_access_memory; + std::unordered_map<GPUVAddr, OGLTransformFeedback> tfb_objects; }; struct BufferCacheParams { diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 339950d2e..7a5fad735 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -309,6 +309,13 @@ void RasterizerOpenGL::DrawIndirect() { const auto& params = maxwell3d->draw_manager->GetIndirectParams(); buffer_cache.SetDrawIndirect(¶ms); PrepareDraw(params.is_indexed, [this, ¶ms](GLenum primitive_mode) { + if (params.is_byte_count) { + const GPUVAddr tfb_object_base_addr = params.indirect_start_address - 4U; + const GLuint tfb_object = + buffer_cache_runtime.GetTransformFeedbackObject(tfb_object_base_addr); + glDrawTransformFeedback(primitive_mode, tfb_object); + return; + } const auto [buffer, offset] = buffer_cache.GetDrawIndirectBuffer(); const GLvoid* const gl_offset = reinterpret_cast<const GLvoid*>(static_cast<uintptr_t>(offset)); @@ -1371,6 +1378,10 @@ void RasterizerOpenGL::ReleaseChannel(s32 channel_id) { query_cache.EraseChannel(channel_id); } +void RasterizerOpenGL::RegisterTransformFeedback(GPUVAddr tfb_object_addr) { + buffer_cache_runtime.BindTransformFeedbackObject(tfb_object_addr); +} + AccelerateDMA::AccelerateDMA(BufferCache& buffer_cache_, TextureCache& texture_cache_) : buffer_cache{buffer_cache_}, texture_cache{texture_cache_} {} diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h index b79d7a70c..ce3460938 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.h +++ b/src/video_core/renderer_opengl/gl_rasterizer.h @@ -139,6 +139,12 @@ public: void ReleaseChannel(s32 channel_id) override; + void RegisterTransformFeedback(GPUVAddr tfb_object_addr) override; + + bool HasDrawTransformFeedback() override { + return true; + } + private: static constexpr size_t MAX_TEXTURES = 192; static constexpr size_t MAX_IMAGES = 48; diff --git a/src/video_core/renderer_opengl/gl_resource_manager.cpp b/src/video_core/renderer_opengl/gl_resource_manager.cpp index eae8fd110..1d2c9b70a 100644 --- a/src/video_core/renderer_opengl/gl_resource_manager.cpp +++ b/src/video_core/renderer_opengl/gl_resource_manager.cpp @@ -207,4 +207,21 @@ void OGLQuery::Release() { handle = 0; } +void OGLTransformFeedback::Create() { + if (handle != 0) + return; + + MICROPROFILE_SCOPE(OpenGL_ResourceCreation); + glCreateTransformFeedbacks(1, &handle); +} + +void OGLTransformFeedback::Release() { + if (handle == 0) + return; + + MICROPROFILE_SCOPE(OpenGL_ResourceDeletion); + glDeleteTransformFeedbacks(1, &handle); + handle = 0; +} + } // namespace OpenGL diff --git a/src/video_core/renderer_opengl/gl_resource_manager.h b/src/video_core/renderer_opengl/gl_resource_manager.h index 77362acd2..6ca8227bd 100644 --- a/src/video_core/renderer_opengl/gl_resource_manager.h +++ b/src/video_core/renderer_opengl/gl_resource_manager.h @@ -323,4 +323,31 @@ public: GLuint handle = 0; }; +class OGLTransformFeedback final { +public: + YUZU_NON_COPYABLE(OGLTransformFeedback); + + OGLTransformFeedback() = default; + + OGLTransformFeedback(OGLTransformFeedback&& o) noexcept : handle(std::exchange(o.handle, 0)) {} + + ~OGLTransformFeedback() { + Release(); + } + + OGLTransformFeedback& operator=(OGLTransformFeedback&& o) noexcept { + Release(); + handle = std::exchange(o.handle, 0); + return *this; + } + + /// Creates a new internal OpenGL resource and stores the handle + void Create(); + + /// Deletes the internal OpenGL resource + void Release(); + + GLuint handle = 0; +}; + } // namespace OpenGL diff --git a/src/video_core/texture_cache/decode_bc.cpp b/src/video_core/texture_cache/decode_bc.cpp index 3e26474a3..a018c6df4 100644 --- a/src/video_core/texture_cache/decode_bc.cpp +++ b/src/video_core/texture_cache/decode_bc.cpp @@ -60,66 +60,72 @@ u32 ConvertedBytesPerBlock(VideoCore::Surface::PixelFormat pixel_format) { } template <auto decompress, PixelFormat pixel_format> -void DecompressBlocks(std::span<const u8> input, std::span<u8> output, Extent3D extent, +void DecompressBlocks(std::span<const u8> input, std::span<u8> output, BufferImageCopy& copy, bool is_signed = false) { const u32 out_bpp = ConvertedBytesPerBlock(pixel_format); - const u32 block_width = std::min(extent.width, BLOCK_SIZE); - const u32 block_height = std::min(extent.height, BLOCK_SIZE); - const u32 pitch = extent.width * out_bpp; + const u32 block_size = BlockSize(pixel_format); + const u32 width = copy.image_extent.width; + const u32 height = copy.image_extent.height * copy.image_subresource.num_layers; + const u32 depth = copy.image_extent.depth; + const u32 block_width = std::min(width, BLOCK_SIZE); + const u32 block_height = std::min(height, BLOCK_SIZE); + const u32 pitch = width * out_bpp; size_t input_offset = 0; size_t output_offset = 0; - for (u32 slice = 0; slice < extent.depth; ++slice) { - for (u32 y = 0; y < extent.height; y += block_height) { - size_t row_offset = 0; - for (u32 x = 0; x < extent.width; - x += block_width, row_offset += block_width * out_bpp) { - const u8* src = input.data() + input_offset; - u8* const dst = output.data() + output_offset + row_offset; + for (u32 slice = 0; slice < depth; ++slice) { + for (u32 y = 0; y < height; y += block_height) { + size_t src_offset = input_offset; + size_t dst_offset = output_offset; + for (u32 x = 0; x < width; x += block_width) { + const u8* src = input.data() + src_offset; + u8* const dst = output.data() + dst_offset; if constexpr (IsSigned(pixel_format)) { - decompress(src, dst, x, y, extent.width, extent.height, is_signed); + decompress(src, dst, x, y, width, height, is_signed); } else { - decompress(src, dst, x, y, extent.width, extent.height); + decompress(src, dst, x, y, width, height); } - input_offset += BlockSize(pixel_format); + src_offset += block_size; + dst_offset += block_width * out_bpp; } + input_offset += copy.buffer_row_length * block_size / block_width; output_offset += block_height * pitch; } } } -void DecompressBCn(std::span<const u8> input, std::span<u8> output, Extent3D extent, +void DecompressBCn(std::span<const u8> input, std::span<u8> output, BufferImageCopy& copy, VideoCore::Surface::PixelFormat pixel_format) { switch (pixel_format) { case PixelFormat::BC1_RGBA_UNORM: case PixelFormat::BC1_RGBA_SRGB: - DecompressBlocks<bcn::DecodeBc1, PixelFormat::BC1_RGBA_UNORM>(input, output, extent); + DecompressBlocks<bcn::DecodeBc1, PixelFormat::BC1_RGBA_UNORM>(input, output, copy); break; case PixelFormat::BC2_UNORM: case PixelFormat::BC2_SRGB: - DecompressBlocks<bcn::DecodeBc2, PixelFormat::BC2_UNORM>(input, output, extent); + DecompressBlocks<bcn::DecodeBc2, PixelFormat::BC2_UNORM>(input, output, copy); break; case PixelFormat::BC3_UNORM: case PixelFormat::BC3_SRGB: - DecompressBlocks<bcn::DecodeBc3, PixelFormat::BC3_UNORM>(input, output, extent); + DecompressBlocks<bcn::DecodeBc3, PixelFormat::BC3_UNORM>(input, output, copy); break; case PixelFormat::BC4_SNORM: case PixelFormat::BC4_UNORM: DecompressBlocks<bcn::DecodeBc4, PixelFormat::BC4_UNORM>( - input, output, extent, pixel_format == PixelFormat::BC4_SNORM); + input, output, copy, pixel_format == PixelFormat::BC4_SNORM); break; case PixelFormat::BC5_SNORM: case PixelFormat::BC5_UNORM: DecompressBlocks<bcn::DecodeBc5, PixelFormat::BC5_UNORM>( - input, output, extent, pixel_format == PixelFormat::BC5_SNORM); + input, output, copy, pixel_format == PixelFormat::BC5_SNORM); break; case PixelFormat::BC6H_SFLOAT: case PixelFormat::BC6H_UFLOAT: DecompressBlocks<bcn::DecodeBc6, PixelFormat::BC6H_UFLOAT>( - input, output, extent, pixel_format == PixelFormat::BC6H_SFLOAT); + input, output, copy, pixel_format == PixelFormat::BC6H_SFLOAT); break; case PixelFormat::BC7_SRGB: case PixelFormat::BC7_UNORM: - DecompressBlocks<bcn::DecodeBc7, PixelFormat::BC7_UNORM>(input, output, extent); + DecompressBlocks<bcn::DecodeBc7, PixelFormat::BC7_UNORM>(input, output, copy); break; default: LOG_WARNING(HW_GPU, "Unimplemented BCn decompression {}", pixel_format); diff --git a/src/video_core/texture_cache/decode_bc.h b/src/video_core/texture_cache/decode_bc.h index 41d1ec0a3..4e3b9b8ac 100644 --- a/src/video_core/texture_cache/decode_bc.h +++ b/src/video_core/texture_cache/decode_bc.h @@ -13,7 +13,7 @@ namespace VideoCommon { [[nodiscard]] u32 ConvertedBytesPerBlock(VideoCore::Surface::PixelFormat pixel_format); -void DecompressBCn(std::span<const u8> input, std::span<u8> output, Extent3D extent, +void DecompressBCn(std::span<const u8> input, std::span<u8> output, BufferImageCopy& copy, VideoCore::Surface::PixelFormat pixel_format); } // namespace VideoCommon diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp index 15596c925..fcf70068e 100644 --- a/src/video_core/texture_cache/util.cpp +++ b/src/video_core/texture_cache/util.cpp @@ -837,6 +837,7 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory std::span<u8> output) { const size_t guest_size_bytes = input.size_bytes(); const u32 bpp_log2 = BytesPerBlockLog2(info.format); + const Extent2D tile_size = DefaultBlockSize(info.format); const Extent3D size = info.size; if (info.type == ImageType::Linear) { @@ -847,7 +848,7 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory return {{ .buffer_offset = 0, .buffer_size = guest_size_bytes, - .buffer_row_length = info.pitch >> bpp_log2, + .buffer_row_length = info.pitch * tile_size.width >> bpp_log2, .buffer_image_height = size.height, .image_subresource = { @@ -862,7 +863,6 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory const LevelInfo level_info = MakeLevelInfo(info); const s32 num_layers = info.resources.layers; const s32 num_levels = info.resources.levels; - const Extent2D tile_size = DefaultBlockSize(info.format); const std::array level_sizes = CalculateLevelSizes(level_info, num_levels); const Extent2D gob = GobSize(bpp_log2, info.block.height, info.tile_width_spacing); const u32 layer_size = CalculateLevelBytes(level_sizes, num_levels); @@ -926,8 +926,6 @@ void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8 const auto input_offset = input.subspan(copy.buffer_offset); copy.buffer_offset = output_offset; - copy.buffer_row_length = mip_size.width; - copy.buffer_image_height = mip_size.height; const auto recompression_setting = Settings::values.astc_recompression.GetValue(); const bool astc = IsPixelFormatASTC(info.format); @@ -972,16 +970,14 @@ void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8 bpp_div; output_offset += static_cast<u32>(copy.buffer_size); } else { - const Extent3D image_extent{ - .width = copy.image_extent.width, - .height = copy.image_extent.height * copy.image_subresource.num_layers, - .depth = copy.image_extent.depth, - }; - DecompressBCn(input_offset, output.subspan(output_offset), image_extent, info.format); + DecompressBCn(input_offset, output.subspan(output_offset), copy, info.format); output_offset += copy.image_extent.width * copy.image_extent.height * copy.image_subresource.num_layers * ConvertedBytesPerBlock(info.format); } + + copy.buffer_row_length = mip_size.width; + copy.buffer_image_height = mip_size.height; } } diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index a6fbca69e..727bbd98d 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -755,10 +755,10 @@ VkFormat Device::GetSupportedFormat(VkFormat wanted_format, VkFormatFeatureFlags // The wanted format is not supported by hardware, search for alternatives const VkFormat* alternatives = GetFormatAlternatives(wanted_format); if (alternatives == nullptr) { - ASSERT_MSG(false, - "Format={} with usage={} and type={} has no defined alternatives and host " - "hardware does not support it", - wanted_format, wanted_usage, format_type); + LOG_ERROR(Render_Vulkan, + "Format={} with usage={} and type={} has no defined alternatives and host " + "hardware does not support it", + wanted_format, wanted_usage, format_type); return wanted_format; } @@ -774,10 +774,10 @@ VkFormat Device::GetSupportedFormat(VkFormat wanted_format, VkFormatFeatureFlags } // No alternatives found, panic - ASSERT_MSG(false, - "Format={} with usage={} and type={} is not supported by the host hardware and " - "doesn't support any of the alternatives", - wanted_format, wanted_usage, format_type); + LOG_ERROR(Render_Vulkan, + "Format={} with usage={} and type={} is not supported by the host hardware and " + "doesn't support any of the alternatives", + wanted_format, wanted_usage, format_type); return wanted_format; } diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index 2f78b8af0..074aed964 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -246,7 +246,9 @@ void SetObjectName(const DeviceDispatch* dld, VkDevice device, T handle, VkObjec .objectHandle = reinterpret_cast<u64>(handle), .pObjectName = name, }; - Check(dld->vkSetDebugUtilsObjectNameEXT(device, &name_info)); + if (dld->vkSetDebugUtilsObjectNameEXT) { + Check(dld->vkSetDebugUtilsObjectNameEXT(device, &name_info)); + } } } // Anonymous namespace diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index 059fcf041..c789c1e59 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -5342,6 +5342,10 @@ int main(int argc, char* argv[]) { if (QString::fromLocal8Bit(qgetenv("DISPLAY")).isEmpty()) { qputenv("DISPLAY", ":0"); } + + // Fix the Wayland appId. This needs to match the name of the .desktop file without the .desktop + // suffix. + QGuiApplication::setDesktopFileName(QStringLiteral("org.yuzu_emu.yuzu")); #endif SetHighDPIAttributes(); diff --git a/src/yuzu/main.h b/src/yuzu/main.h index 530e445f9..366e806d5 100644 --- a/src/yuzu/main.h +++ b/src/yuzu/main.h @@ -168,14 +168,6 @@ class GMainWindow : public QMainWindow { /// Max number of recently loaded items to keep track of static const int max_recent_files_item = 10; - // TODO: Make use of this! - enum { - UI_IDLE, - UI_EMU_BOOTING, - UI_EMU_RUNNING, - UI_EMU_STOPPING, - }; - enum { CREATE_SHORTCUT_MSGBOX_FULLSCREEN_YES, CREATE_SHORTCUT_MSGBOX_SUCCESS, diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl2.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl2.cpp index 5153cdb79..1a35d471c 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl2.cpp +++ b/src/yuzu_cmd/emu_window/emu_window_sdl2.cpp @@ -20,7 +20,7 @@ EmuWindow_SDL2::EmuWindow_SDL2(InputCommon::InputSubsystem* input_subsystem_, Co : input_subsystem{input_subsystem_}, system{system_} { input_subsystem->Initialize(); if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_JOYSTICK | SDL_INIT_GAMECONTROLLER) < 0) { - LOG_CRITICAL(Frontend, "Failed to initialize SDL2! Exiting..."); + LOG_CRITICAL(Frontend, "Failed to initialize SDL2: {}, Exiting...", SDL_GetError()); exit(1); } SDL_SetMainReady(); diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl2_vk.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl2_vk.cpp index 9ed47d453..8b916f05c 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl2_vk.cpp +++ b/src/yuzu_cmd/emu_window/emu_window_sdl2_vk.cpp @@ -28,7 +28,8 @@ EmuWindow_SDL2_VK::EmuWindow_SDL2_VK(InputCommon::InputSubsystem* input_subsyste SDL_SysWMinfo wm; SDL_VERSION(&wm.version); if (SDL_GetWindowWMInfo(render_window, &wm) == SDL_FALSE) { - LOG_CRITICAL(Frontend, "Failed to get information from the window manager"); + LOG_CRITICAL(Frontend, "Failed to get information from the window manager: {}", + SDL_GetError()); std::exit(EXIT_FAILURE); } |